diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 68bbc7315f90..6e92367f5244 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2237,6 +2237,29 @@ Kernel parameters idle=nomwait: Disable mwait for CPU C-states + [ARM64,EARLY] + Format: idle=wfi, idle=yield, idle=nop + + idle=wfi: Use the WFI (Wait For Interrupt) hint + instruction in the idle loop. This is the default and + allows the CPU to enter a low-power state until an + interrupt arrives. + + idle=yield: Use the YIELD hint instruction instead of + WFI. CPUs supporting simultaneous multi-threading (SMT), + can continue executing another thread when the current + thread reaches the idle loop. This will make the CPUs + eat more power, but may be useful to get slightly better + performance in some applications, since the CPUs will + not enter a low-power state. + + idle=nop: Do not execute any idle instruction in the + idle loop. This is useful on platforms where WFI + misbehaves, leading to system instability or loss of CPU + state. This will make the CPUs eat more power, but may + give slightly better performance in some applications, + since the CPUs will not enter a low-power state. + idxd.sva= [HW] Format: Allow force disabling of Shared Virtual Memory (SVA) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index add66afc7b03..f603afe323d6 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -113,240 +113,375 @@ infrastructure: 4. List of registers with visible features ------------------------------------------- - 1) ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 + ID_AA64FPFR0_EL1 - Floating Point feature ID register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | RNDR | [63-60] | y | - +------------------------------+---------+---------+ - | TS | [55-52] | y | - +------------------------------+---------+---------+ - | FHM | [51-48] | y | - +------------------------------+---------+---------+ - | DP | [47-44] | y | - +------------------------------+---------+---------+ - | SM4 | [43-40] | y | - +------------------------------+---------+---------+ - | SM3 | [39-36] | y | - +------------------------------+---------+---------+ - | SHA3 | [35-32] | y | - +------------------------------+---------+---------+ - | RDM | [31-28] | y | - +------------------------------+---------+---------+ - | ATOMICS | [23-20] | y | - +------------------------------+---------+---------+ - | CRC32 | [19-16] | y | - +------------------------------+---------+---------+ - | SHA2 | [15-12] | y | - +------------------------------+---------+---------+ - | SHA1 | [11-8] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | F8CVT | [31] | + +------------------------------+---------+ + | F8FMA | [30] | + +------------------------------+---------+ + | F8DP4 | [29] | + +------------------------------+---------+ + | F8DP2 | [28] | + +------------------------------+---------+ + | F8MM8 | [27] | + +------------------------------+---------+ + | F8MM4 | [26] | + +------------------------------+---------+ + | F16MM2 | [15] | + +------------------------------+---------+ + | F8E4M3 | [1] | + +------------------------------+---------+ + | F8E5M2 | [0] | + +------------------------------+---------+ + + ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | RNDR | [63-60] | + +------------------------------+---------+ + | TS | [55-52] | + +------------------------------+---------+ + | FHM | [51-48] | + +------------------------------+---------+ + | DP | [47-44] | + +------------------------------+---------+ + | SM4 | [43-40] | + +------------------------------+---------+ + | SM3 | [39-36] | + +------------------------------+---------+ + | SHA3 | [35-32] | + +------------------------------+---------+ + | RDM | [31-28] | + +------------------------------+---------+ + | ATOMICS | [23-20] | + +------------------------------+---------+ + | CRC32 | [19-16] | + +------------------------------+---------+ + | SHA2 | [15-12] | + +------------------------------+---------+ + | SHA1 | [11-8] | + +------------------------------+---------+ + | AES | [7-4] | + +------------------------------+---------+ - 2) ID_AA64PFR0_EL1 - Processor Feature Register 0 + ID_AA64ISAR1_EL1 - Instruction set attribute register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | DIT | [51-48] | y | - +------------------------------+---------+---------+ - | MPAM | [43-40] | n | - +------------------------------+---------+---------+ - | SVE | [35-32] | y | - +------------------------------+---------+---------+ - | GIC | [27-24] | n | - +------------------------------+---------+---------+ - | AdvSIMD | [23-20] | y | - +------------------------------+---------+---------+ - | FP | [19-16] | y | - +------------------------------+---------+---------+ - | EL3 | [15-12] | n | - +------------------------------+---------+---------+ - | EL2 | [11-8] | n | - +------------------------------+---------+---------+ - | EL1 | [7-4] | n | - +------------------------------+---------+---------+ - | EL0 | [3-0] | n | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | LS64 | [63-60] | + +------------------------------+---------+ + | I8MM | [55-52] | + +------------------------------+---------+ + | DGH | [51-48] | + +------------------------------+---------+ + | BF16 | [47-44] | + +------------------------------+---------+ + | SB | [39-36] | + +------------------------------+---------+ + | FRINTTS | [35-32] | + +------------------------------+---------+ + | GPI | [31-28] | + +------------------------------+---------+ + | GPA | [27-24] | + +------------------------------+---------+ + | LRCPC | [23-20] | + +------------------------------+---------+ + | FCMA | [19-16] | + +------------------------------+---------+ + | JSCVT | [15-12] | + +------------------------------+---------+ + | API | [11-8] | + +------------------------------+---------+ + | APA | [7-4] | + +------------------------------+---------+ + | DPB | [3-0] | + +------------------------------+---------+ + + ID_AA64ISAR2_EL1 - Instruction set attribute register 2 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | LUT | [59-56] | + +------------------------------+---------+ + | CSSC | [55-52] | + +------------------------------+---------+ + | RPRFM | [51-48] | + +------------------------------+---------+ + | BC | [23-20] | + +------------------------------+---------+ + | MOPS | [19-16] | + +------------------------------+---------+ + | APA3 | [15-12] | + +------------------------------+---------+ + | GPA3 | [11-8] | + +------------------------------+---------+ + | RPRES | [7-4] | + +------------------------------+---------+ + | WFXT | [3-0] | + +------------------------------+---------+ + + ID_AA64ISAR3_EL1 - Instruction set attribute register 3 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPRCVT | [31-28] | + +------------------------------+---------+ + | LSFE | [19-16] | + +------------------------------+---------+ + | FAMINMAX | [7-4] | + +------------------------------+---------+ + + ID_AA64MMFR0_EL1 - Memory model feature register 0 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | ECV | [63-60] | + +------------------------------+---------+ + + ID_AA64MMFR1_EL1 - Memory model feature register 1 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | AFP | [47-44] | + +------------------------------+---------+ + + ID_AA64MMFR2_EL1 - Memory model feature register 2 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | AT | [35-32] | + +------------------------------+---------+ + + ID_AA64MMFR3_EL1 - Memory model feature register 3 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | S1POE | [19-16] | + +------------------------------+---------+ + + ID_AA64PFR0_EL1 - Processor Feature Register 0 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | DIT | [51-48] | + +------------------------------+---------+ + | SVE | [35-32] | + +------------------------------+---------+ + | AdvSIMD | [23-20] | + +------------------------------+---------+ + | FP | [19-16] | + +------------------------------+---------+ - 3) ID_AA64PFR1_EL1 - Processor Feature Register 1 + ID_AA64PFR1_EL1 - Processor Feature Register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | SME | [27-24] | y | - +------------------------------+---------+---------+ - | MTE | [11-8] | y | - +------------------------------+---------+---------+ - | SSBS | [7-4] | y | - +------------------------------+---------+---------+ - | BT | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | GCS | [47-44] | + +------------------------------+---------+ + | SME | [27-24] | + +------------------------------+---------+ + | MTE | [11-8] | + +------------------------------+---------+ + | SSBS | [7-4] | + +------------------------------+---------+ + | BT | [3-0] | + +------------------------------+---------+ + ID_AA64PFR2_EL1 - Processor Feature Register 2 - 4) MIDR_EL1 - Main ID Register + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPMR | [35-32] | + +------------------------------+---------+ + | MTEFAR | [11-8] | + +------------------------------+---------+ + | MTESTOREONLY | [7-4] | + +------------------------------+---------+ - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | Implementer | [31-24] | y | - +------------------------------+---------+---------+ - | Variant | [23-20] | y | - +------------------------------+---------+---------+ - | Architecture | [19-16] | y | - +------------------------------+---------+---------+ - | PartNum | [15-4] | y | - +------------------------------+---------+---------+ - | Revision | [3-0] | y | - +------------------------------+---------+---------+ + ID_AA64SMFR0_EL1 - SME feature ID register 0 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FA64 | [63] | + +------------------------------+---------+ + | LUT6 | [61] | + +------------------------------+---------+ + | LUTv2 | [60] | + +------------------------------+---------+ + | SMEver | [59-56] | + +------------------------------+---------+ + | I16I64 | [55-52] | + +------------------------------+---------+ + | F64F64 | [48] | + +------------------------------+---------+ + | I16I32 | [47-44] | + +------------------------------+---------+ + | B16B16 | [43] | + +------------------------------+---------+ + | F16F16 | [42] | + +------------------------------+---------+ + | F8F16 | [41] | + +------------------------------+---------+ + | F8F32 | [40] | + +------------------------------+---------+ + | I8I32 | [39-36] | + +------------------------------+---------+ + | F16F32 | [35] | + +------------------------------+---------+ + | B16F32 | [34] | + +------------------------------+---------+ + | BI32I32 | [33] | + +------------------------------+---------+ + | F32F32 | [32] | + +------------------------------+---------+ + | SF8FMA | [30] | + +------------------------------+---------+ + | SF8DP4 | [29] | + +------------------------------+---------+ + | SF8DP2 | [28] | + +------------------------------+---------+ + | SBitPerm | [25] | + +------------------------------+---------+ + | AES | [24] | + +------------------------------+---------+ + | SFEXPA | [23] | + +------------------------------+---------+ + | STMOP | [16] | + +------------------------------+---------+ + | SMOP4 | [0] | + +------------------------------+---------+ + + ID_AA64ZFR0_EL1 - SVE feature ID register 0 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | F64MM | [59-56] | + +------------------------------+---------+ + | F32MM | [55-52] | + +------------------------------+---------+ + | F16MM | [51-48] | + +------------------------------+---------+ + | I8MM | [47-44] | + +------------------------------+---------+ + | SM4 | [43-40] | + +------------------------------+---------+ + | SHA3 | [35-32] | + +------------------------------+---------+ + | B16B16 | [27-24] | + +------------------------------+---------+ + | BF16 | [23-20] | + +------------------------------+---------+ + | BitPerm | [19-16] | + +------------------------------+---------+ + | EltPerm | [15-12] | + +------------------------------+---------+ + | AES | [7-4] | + +------------------------------+---------+ + | SVEVer | [3-0] | + +------------------------------+---------+ + + ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | CRC32 | [19-16] | + +------------------------------+---------+ + | SHA2 | [15-12] | + +------------------------------+---------+ + | SHA1 | [11-8] | + +------------------------------+---------+ + | AES | [7-4] | + +------------------------------+---------+ + + ID_ISAR6_EL1 - AArch32 Instruction Set Attribute Register 6 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | I8MM | [27-24] | + +------------------------------+---------+ + | BF16 | [23-20] | + +------------------------------+---------+ + | SB | [15-12] | + +------------------------------+---------+ + | FHM | [11-8] | + +------------------------------+---------+ + | DP | [7-4] | + +------------------------------+---------+ + + ID_PFR2_EL1 - AArch32 Processor Feature Register 2 + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | SSBS | [7-4] | + +------------------------------+---------+ + + MIDR_EL1 - Main ID Register + + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | Implementer | [31-24] | + +------------------------------+---------+ + | Variant | [23-20] | + +------------------------------+---------+ + | Architecture | [19-16] | + +------------------------------+---------+ + | PartNum | [15-4] | + +------------------------------+---------+ + | Revision | [3-0] | + +------------------------------+---------+ NOTE: The 'visible' fields of MIDR_EL1 will contain the value as available on the CPU where it is fetched and is not a system wide safe value. - 5) ID_AA64ISAR1_EL1 - Instruction set attribute register 1 + MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | I8MM | [55-52] | y | - +------------------------------+---------+---------+ - | DGH | [51-48] | y | - +------------------------------+---------+---------+ - | BF16 | [47-44] | y | - +------------------------------+---------+---------+ - | SB | [39-36] | y | - +------------------------------+---------+---------+ - | FRINTTS | [35-32] | y | - +------------------------------+---------+---------+ - | GPI | [31-28] | y | - +------------------------------+---------+---------+ - | GPA | [27-24] | y | - +------------------------------+---------+---------+ - | LRCPC | [23-20] | y | - +------------------------------+---------+---------+ - | FCMA | [19-16] | y | - +------------------------------+---------+---------+ - | JSCVT | [15-12] | y | - +------------------------------+---------+---------+ - | API | [11-8] | y | - +------------------------------+---------+---------+ - | APA | [7-4] | y | - +------------------------------+---------+---------+ - | DPB | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPDP | [11-8] | + +------------------------------+---------+ - 6) ID_AA64MMFR0_EL1 - Memory model feature register 0 + MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | ECV | [63-60] | y | - +------------------------------+---------+---------+ - - 7) ID_AA64MMFR2_EL1 - Memory model feature register 2 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AT | [35-32] | y | - +------------------------------+---------+---------+ - - 8) ID_AA64ZFR0_EL1 - SVE feature ID register 0 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | F64MM | [59-56] | y | - +------------------------------+---------+---------+ - | F32MM | [55-52] | y | - +------------------------------+---------+---------+ - | I8MM | [47-44] | y | - +------------------------------+---------+---------+ - | SM4 | [43-40] | y | - +------------------------------+---------+---------+ - | SHA3 | [35-32] | y | - +------------------------------+---------+---------+ - | B16B16 | [27-24] | y | - +------------------------------+---------+---------+ - | BF16 | [23-20] | y | - +------------------------------+---------+---------+ - | BitPerm | [19-16] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | - +------------------------------+---------+---------+ - | SVEVer | [3-0] | y | - +------------------------------+---------+---------+ - - 8) ID_AA64MMFR1_EL1 - Memory model feature register 1 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AFP | [47-44] | y | - +------------------------------+---------+---------+ - - 9) ID_AA64ISAR2_EL1 - Instruction set attribute register 2 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | CSSC | [55-52] | y | - +------------------------------+---------+---------+ - | RPRFM | [51-48] | y | - +------------------------------+---------+---------+ - | BC | [23-20] | y | - +------------------------------+---------+---------+ - | MOPS | [19-16] | y | - +------------------------------+---------+---------+ - | APA3 | [15-12] | y | - +------------------------------+---------+---------+ - | GPA3 | [11-8] | y | - +------------------------------+---------+---------+ - | RPRES | [7-4] | y | - +------------------------------+---------+---------+ - | WFXT | [3-0] | y | - +------------------------------+---------+---------+ - - 10) MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPDP | [11-8] | y | - +------------------------------+---------+---------+ - - 11) MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | SIMDFMAC | [31-28] | y | - +------------------------------+---------+---------+ - | SIMDSP | [19-16] | y | - +------------------------------+---------+---------+ - | SIMDInt | [15-12] | y | - +------------------------------+---------+---------+ - | SIMDLS | [11-8] | y | - +------------------------------+---------+---------+ - - 12) ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | CRC32 | [19-16] | y | - +------------------------------+---------+---------+ - | SHA2 | [15-12] | y | - +------------------------------+---------+---------+ - | SHA1 | [11-8] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | SIMDFMAC | [31-28] | + +------------------------------+---------+ + | FPHP | [27-24] | + +------------------------------+---------+ + | SIMDHP | [23-20] | + +------------------------------+---------+ + | SIMDSP | [19-16] | + +------------------------------+---------+ + | SIMDInt | [15-12] | + +------------------------------+---------+ + | SIMDLS | [11-8] | + +------------------------------+---------+ Appendix I: Example diff --git a/Documentation/arch/arm64/mpam.rst b/Documentation/arch/arm64/mpam.rst index 570f51a8d4eb..67fe515ed501 100644 --- a/Documentation/arch/arm64/mpam.rst +++ b/Documentation/arch/arm64/mpam.rst @@ -65,6 +65,28 @@ The supported features are: there is at least one CSU monitor on each MSC that makes up the L3 group. Exposing CSU counters from other caches or devices is not supported. +* Memory Bandwidth Usage (MBWU) on or after the L3 cache. resctrl uses the + L3 cache-id to identify where the memory bandwidth is measured. For this + reason the platform must have an L3 cache with cache-id's supplied by + firmware. (The platform doesn't need to support MPAM.) + + Memory bandwidth monitoring makes use of MBWU monitors in each MSC that + makes up the L3 group. If the memory bandwidth monitoring is on the memory + rather than the L3 then there must be a single global L3 as otherwise it + is unknown which L3 the traffic came from. + + To expose 'mbm_total_bytes', the topology of the group of MSC chosen must + match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth monitors on + CPU-less NUMA nodes cannot expose 'mbm_total_bytes' as these nodes do not + have a corresponding L3 cache. 'mbm_local_bytes' is not exposed as MPAM + cannot distinguish local traffic from global traffic. + + All these restrictions based on L3 cache are due to resctrl, currently, only + supporting monitoring at the L3 scope. It is expected that going forward more + MBWU monitors can be exposed to the user after support for more monitoring + scopes is added to resctrl. + Reporting Bugs ============== If you are not seeing the counters or controls you expect please share the diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 88b4aa45a206..68acd6555661 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -246,10 +246,14 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #3683289 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #3683289 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| @@ -308,6 +312,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1027 | NVIDIA_OLYMPUS_1027_ERRATUM | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | diff --git a/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml b/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml index a18dd0a8c43a..f2f0d6b61eac 100644 --- a/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml +++ b/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml @@ -4,16 +4,22 @@ $id: http://devicetree.org/schemas/perf/marvell-cn10k-ddr.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Marvell CN10K DDR performance monitor +title: Marvell CN10K / CN20K DDR performance monitor + +description: + Performance Monitoring Unit (PMU) for the DDR controller on Marvell + CN10K and CN20K SoCs. The block is accessed via a dedicated MMIO region. maintainers: - Bharat Bhushan + - Geetha sowjanya properties: compatible: items: - enum: - marvell,cn10k-ddr-pmu + - marvell,cn20k-ddr-pmu reg: maxItems: 1 diff --git a/MAINTAINERS b/MAINTAINERS index 1d93f008fb33..8f4f9f117130 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3904,6 +3904,7 @@ F: drivers/platform/arm64/ ARM64 PORT (AARCH64 ARCHITECTURE) M: Catalin Marinas M: Will Deacon +R: Mark Rutland L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git @@ -25084,7 +25085,7 @@ M: James Morse L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: Documentation/devicetree/bindings/arm/firmware/sdei.txt -F: drivers/firmware/arm_sdei.c +F: drivers/firmware/arm_sdei* F: include/linux/arm_sdei.h F: include/uapi/linux/arm_sdei.h diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index 847590df7551..d4ac8d3271b1 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -246,11 +246,7 @@ static inline void gic_pmr_mask_irqs(void) WARN_ON_ONCE(true); } -static inline void gic_arch_enable_irqs(void) -{ - /* Should not get called. */ - WARN_ON_ONCE(true); -} +static inline void gic_unmask_pnmis(void) {} static inline bool gic_has_relaxed_pmr_sync(void) { diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..8b863b9f2124 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -187,7 +187,7 @@ config ARM64 if (GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS || \ CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS) select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS \ - if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS + if DYNAMIC_FTRACE_WITH_ARGS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \ if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE)) @@ -701,12 +701,34 @@ config ARM64_ERRATUM_1530923 If unsure, say Y. -config ARM64_WORKAROUND_REPEAT_TLBI +config ARM64_WORKAROUND_REPEAT_TLBI_SYNC bool + # This workaround is (only) suitable for TLB invalidation errata where + # all of the following conditions are true: + # + # - The effects of the errata are only a loss of ordering/completion + # for explicit memory accesses when the TLBI is completed with a DSB. + # The removal of TLB entries is not affected. + # + # Note that architecturally, S2-only invalidation does not remove + # combined S1+S2 entries, and does not complete accesses translated + # via those S1+S2 entries. Consequently, where this condition holds, + # the errata do not affect S2-only invalidation. + # + # - The errata only affect broadcast TLB invalidation operations (e.g. + # TLBI VMALLE1IS), and do not affect local TLB invalidation + # operations (e.g. TLBI VMALLE1). + # + # - After any number of affected TLBI operations are completed with a + # DSB, the errata can be mitigated by executing a single arbitrary + # broadcast TLBI (which targets an arbitrary translation regime), + # followed by a DSB. + # + # For more rationale, see commit a8f78680ee6bf795. config ARM64_ERRATUM_2441007 bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A55 erratum #2441007. @@ -722,7 +744,7 @@ config ARM64_ERRATUM_2441007 config ARM64_ERRATUM_1286807 bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A76 erratum 1286807. @@ -944,7 +966,7 @@ config ARM64_ERRATUM_2224489 config ARM64_ERRATUM_2441009 bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A510 erratum #2441009. @@ -1156,7 +1178,7 @@ config ARM64_ERRATUM_4193714 config ARM64_ERRATUM_4118414 bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for the following errata: @@ -1340,7 +1362,7 @@ config QCOM_FALKOR_ERRATUM_1003 config QCOM_FALKOR_ERRATUM_1009 bool "Falkor E1009: Prematurely complete a DSB after a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help On Falkor v1, the CPU may prematurely complete a DSB following a TLBI xxIS invalidate maintenance operation. Repeat the TLBI operation @@ -1382,6 +1404,28 @@ config NVIDIA_CARMEL_CNP_ERRATUM If unsure, say Y. +config NVIDIA_OLYMPUS_1027_ERRATUM + bool "NVIDIA Olympus: device store/load ordering erratum" + default y + help + This option adds an alternative code sequence to work around an + NVIDIA Olympus core erratum where a Device-nGnR* store can be + observed by a peripheral after a younger Device-nGnR* load to the + same peripheral. This breaks the program order that drivers rely + on for MMIO and can leave a device in an incorrect state. + + The workaround inserts a DMB OSH immediately before raw MMIO loads. + The erratum cannot occur when a DMB that orders loads appears + between the store and load, preventing the younger load from being + observed before the older store. + + The alternatives framework patches in DMB OSH only when an affected + CPU is detected. Other CPUs execute a NOP in its place. Disabling + this option leaves the original MMIO read instruction stream + unchanged. + + If unsure, say Y. + config ROCKCHIP_ERRATUM_3568002 bool "Rockchip 3568002: GIC600 can not access physical addresses higher than 4GB" default y @@ -2116,6 +2160,8 @@ config ARM64_BTI_KERNEL depends on !CC_IS_GCC || GCC_VERSION >= 100100 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 depends on !CC_IS_GCC + # https://github.com/llvm/llvm-project/issues/215547 + depends on !CC_IS_CLANG || CLANG_VERSION < 210000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS) help Build the kernel with Branch Target Identification annotations diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index d20b03931a8d..3dcb7b8309d9 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -178,9 +178,12 @@ static inline void gic_pmr_mask_irqs(void) gic_write_pmr(GIC_PRIO_IRQOFF); } -static inline void gic_arch_enable_irqs(void) +static inline void gic_unmask_pnmis(void) { - asm volatile ("msr daifclr, #3" : : : "memory"); + if (gic_prio_masking_enabled()) { + gic_pmr_mask_irqs(); + asm volatile ("msr daifclr, #3" : : : "memory"); + } } static inline bool gic_has_relaxed_pmr_sync(void) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index effae53e9739..0b58b550e8dc 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -34,10 +34,6 @@ wx\n .req w\n .endr - .macro disable_daif - msr daifset, #0xf - .endm - /* * Save/restore interrupts. */ diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 25c61cda901c..76350b38f0d7 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -60,8 +60,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); case ARM64_WORKAROUND_DISABLE_CNP: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); - case ARM64_WORKAROUND_REPEAT_TLBI: - return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); + case ARM64_WORKAROUND_REPEAT_TLBI_SYNC: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); case ARM64_WORKAROUND_4193714: diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db..d90040fb9de6 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -878,11 +878,11 @@ static inline bool system_supports_pmuv3(void) return cpus_have_final_cap(ARM64_HAS_PMUV3); } -bool cpu_supports_bbml2_noabort(void); +bool cpu_supports_bbml3(void); -static inline bool system_supports_bbml2_noabort(void) +static inline bool system_supports_bbml3(void) { - return alternative_has_cap_unlikely(ARM64_HAS_BBML2_NOABORT); + return alternative_has_cap_unlikely(ARM64_HAS_BBML3); } int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 1b9f0cda1336..1fa29616e586 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -82,6 +82,7 @@ #define ARM_CPU_PART_CORTEX_X1 0xD44 #define ARM_CPU_PART_CORTEX_A510 0xD46 #define ARM_CPU_PART_CORTEX_A520 0xD80 +#define ARM_CPU_PART_CORTEX_A520AE 0xD88 #define ARM_CPU_PART_CORTEX_A710 0xD47 #define ARM_CPU_PART_CORTEX_A715 0xD4D #define ARM_CPU_PART_CORTEX_X2 0xD48 @@ -99,6 +100,7 @@ #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_NANO 0xD8A #define ARM_CPU_PART_C1_PRO 0xD8B #define ARM_CPU_PART_C1_PREMIUM 0xD90 @@ -176,6 +178,7 @@ #define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) #define MIDR_CORTEX_A520 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520) +#define MIDR_CORTEX_A520AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520AE) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) #define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715) #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) @@ -193,6 +196,7 @@ #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_NANO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_NANO) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h index cab8cd78f693..1905765159aa 100644 --- a/arch/arm64/include/asm/entry-common.h +++ b/arch/arm64/include/asm/entry-common.h @@ -32,7 +32,7 @@ static inline bool arch_irqentry_exit_need_resched(void) /* * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC * priority masking is used the GIC irqchip driver will clear DAIF.IF - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * in gic_unmask_pnmis() for normal IRQs. If anything is set in * DAIF we must have handled an NMI, so skip preemption. */ if (system_uses_irq_prio_masking() && read_sysreg(daif)) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index d1d2ff9d323a..79c6d86c38a9 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -151,42 +151,31 @@ __lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) } static __always_inline int -__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +__lsui_cmpxchg32(u32 __user *uaddr, u32 *oldval, u32 newval) { u64 __user *uaddr64; bool futex_pos, other_pos; - u32 other, orig_other; union { u32 futex[2]; u64 raw; - } oval64, orig64, nval64; + } orig64, oval64, nval64; uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64)); futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); other_pos = !futex_pos; - oval64.futex[futex_pos] = oldval; - if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) + orig64.futex[futex_pos] = *oldval; + if (get_user(orig64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) return -EFAULT; - orig64.raw = oval64.raw; - + nval64 = oval64 = orig64; nval64.futex[futex_pos] = newval; - nval64.futex[other_pos] = oval64.futex[other_pos]; if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) return -EFAULT; - oldval = oval64.futex[futex_pos]; - other = oval64.futex[other_pos]; - orig_other = orig64.futex[other_pos]; - - if (other != orig_other) - return -EAGAIN; - - *oval = oldval; - - return 0; + *oldval = oval64.futex[futex_pos]; + return oval64.raw == orig64.raw ? 0 : -EAGAIN; } static __always_inline int @@ -202,7 +191,7 @@ __lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) static __always_inline int __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) { - u32 oldval, newval, val; + u32 oldval, newval; int ret, i; if (get_user(oldval, uaddr)) @@ -214,33 +203,27 @@ __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) for (i = 0; i < FUTEX_MAX_LOOPS; i++) { newval = oldval ^ oparg; - ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val); - switch (ret) { - case -EFAULT: - return ret; - case -EAGAIN: - continue; - } - - if (val == oldval) { - *oval = val; - return 0; - } - - oldval = val; + ret = __lsui_cmpxchg32(uaddr, &oldval, newval); + if (ret != -EAGAIN) + break; } - return -EAGAIN; + *oval = oldval; + return ret; } static __always_inline int __lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) { - /* - * Callers of futex_atomic_cmpxchg_inatomic() already retry on - * -EAGAIN, no need for another loop of max retries. - */ - return __lsui_cmpxchg32(uaddr, oldval, newval, oval); + u32 curval = oldval; + int ret; + + ret = __lsui_cmpxchg32(uaddr, &curval, newval); + if (ret == -EAGAIN && curval != oldval) + ret = 0; + + *oval = curval; + return ret; } #endif /* CONFIG_ARM64_LSUI */ diff --git a/arch/arm64/include/asm/hypervisor.h b/arch/arm64/include/asm/hypervisor.h index a12fd897c877..8889a0ba1ec5 100644 --- a/arch/arm64/include/asm/hypervisor.h +++ b/arch/arm64/include/asm/hypervisor.h @@ -3,6 +3,9 @@ #define _ASM_ARM64_HYPERVISOR_H #include +#include + +DECLARE_STATIC_KEY_FALSE(pkvm_guest); void kvm_init_hyp_services(void); bool kvm_arm_hyp_service_available(u32 func_id); @@ -10,8 +13,18 @@ void kvm_arm_target_impl_cpu_init(void); #ifdef CONFIG_ARM_PKVM_GUEST void pkvm_init_hyp_services(void); + +static inline bool is_protected_kvm_guest(void) +{ + return static_branch_unlikely(&pkvm_guest); +} #else static inline void pkvm_init_hyp_services(void) { }; + +static inline bool is_protected_kvm_guest(void) +{ + return false; +} #endif static inline void kvm_arch_init_hyp_services(void) diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 21c8e400107c..49a7002661a9 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -54,7 +54,9 @@ static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr) static __always_inline u8 __raw_readb(const volatile void __iomem *addr) { u8 val; - asm volatile(ALTERNATIVE("ldrb %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldrb %w0, [%1]", "ldarb %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -66,7 +68,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr) { u16 val; - asm volatile(ALTERNATIVE("ldrh %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldrh %w0, [%1]", "ldarh %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -77,7 +81,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr) static __always_inline u32 __raw_readl(const volatile void __iomem *addr) { u32 val; - asm volatile(ALTERNATIVE("ldr %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldr %w0, [%1]", "ldar %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -88,7 +94,9 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr) static __always_inline u64 __raw_readq(const volatile void __iomem *addr) { u64 val; - asm volatile(ALTERNATIVE("ldr %0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldr %0, [%1]", "ldar %0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h index 314b2b52025f..636f45b4d8af 100644 --- a/arch/arm64/include/asm/mem_encrypt.h +++ b/arch/arm64/include/asm/mem_encrypt.h @@ -2,6 +2,7 @@ #ifndef __ASM_MEM_ENCRYPT_H #define __ASM_MEM_ENCRYPT_H +#include #include struct device; @@ -20,7 +21,7 @@ int realm_register_memory_enc_ops(void); static inline bool force_dma_unencrypted(struct device *dev) { - return is_realm_world(); + return is_realm_world() || is_protected_kvm_guest(); } /* diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h new file mode 100644 index 000000000000..2e8974ff8d63 --- /dev/null +++ b/arch/arm64/include/asm/nmi.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NMI_H +#define __ASM_NMI_H + +#include + +struct pt_regs; + +/* + * Cross-CPU NMI provider hooks, consulted by the arm64 arch code before + * its regular-IRQ / pseudo-NMI IPI paths. The SDEI provider in + * drivers/firmware/arm_sdei_nmi.c implements them when active; a future + * FEAT_NMI provider could slot in here too. The stubs let callers stay + * unconditional when ARM_SDEI_NMI is off. + * + * sdei_nmi_active() lets a caller test for the service before committing + * to (and waiting on) the SDEI stop rung; sdei_nmi_stop_cpus() then signals + * the targets, which ack by going offline. + */ +#ifdef CONFIG_ARM_SDEI_NMI +bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); +bool sdei_nmi_active(void); +void sdei_nmi_stop_cpus(const cpumask_t *mask); +#else +static inline bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, + int exclude_cpu) +{ + return false; +} + +static inline bool sdei_nmi_active(void) +{ + return false; +} + +static inline void sdei_nmi_stop_cpus(const cpumask_t *mask) { } +#endif + +/* + * The common "stop this CPU" entry every arm64 stop path funnels through: + * the regular/pseudo-NMI stop IPI handlers, panic_smp_self_stop(), and the + * SDEI cross-CPU NMI handler. @die_on_crash powers the CPU off on the kdump + * crash path (IPI handlers) instead of parking it (SDEI / self-stop). + * Defined in arch/arm64/kernel/smp.c. + */ +void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash); + +#endif /* __ASM_NMI_H */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..25001694ae8e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -140,10 +140,17 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_none(pte) (!pte_val(pte)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +#define pte_present_invalid(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) + /* * The following only work if pte_present(). Undefined behaviour otherwise. */ -#define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte)) +static __always_inline bool pte_present(pte_t pte) +{ + return pte_valid(pte) || pte_present_invalid(pte); +} #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) @@ -168,9 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) -#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) -#define pte_present_invalid(pte) \ - ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 39582511ad72..f7dc5fb9427d 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -18,8 +18,6 @@ #define INIT_PSTATE_EL1 \ (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h) -#define INIT_PSTATE_EL2 \ - (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h) #include diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index 2c8763876dfb..c1fab41f671e 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, return res.a0; } +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + /** * rsi_attestation_token_init - Initialise the operation to retrieve an * attestation token. @@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, static inline long rsi_attestation_token_init(const u8 *challenge, unsigned long size) { - struct arm_smccc_1_2_regs regs = { 0 }; + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; - /* The challenge must be at least 32bytes and at most 64bytes */ - if (!challenge || size < 32 || size > 64) + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) return -EINVAL; - regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; - memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); - if (regs.a0 == RSI_SUCCESS) - return regs.a1; + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; return -EINVAL; } diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h index abb57bc54305..395152ef5372 100644 --- a/arch/arm64/include/asm/syscall_wrapper.h +++ b/arch/arm64/include/asm/syscall_wrapper.h @@ -10,13 +10,13 @@ #include -#define SC_ARM64_REGS_TO_ARGS(x, ...) \ +#ifdef CONFIG_COMPAT + +#define COMPAT_SC_ARM64_REGS_TO_ARGS(x, ...) \ __MAP(x,__SC_ARGS \ ,,regs->regs[0],,regs->regs[1],,regs->regs[2] \ ,,regs->regs[3],,regs->regs[4],,regs->regs[5]) -#ifdef CONFIG_COMPAT - #define COMPAT_SYSCALL_DEFINEx(x, name, ...) \ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs); \ ALLOW_ERROR_INJECTION(__arm64_compat_sys##name, ERRNO); \ @@ -24,7 +24,7 @@ static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs) \ { \ - return __se_compat_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ + return __se_compat_sys##name(COMPAT_SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ } \ static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ { \ @@ -46,6 +46,11 @@ #endif /* CONFIG_COMPAT */ +#define SC_ARM64_REGS_TO_ARGS(x, ...) \ + __MAP(x,__SC_ARGS \ + ,,regs->orig_x0,,regs->regs[1],,regs->regs[2] \ + ,,regs->regs[3],,regs->regs[4],,regs->regs[5]) + #define __SYSCALL_DEFINEx(x, name, ...) \ asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \ ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index e0e84332f51b..14a78ac0f800 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -236,7 +236,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) #define __repeat_tlbi_sync(op, arg...) \ do { \ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI_SYNC)) \ break; \ __tlbi(op, ##arg); \ dsb(ish); \ diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 6fed93fb2536..15649a253a57 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -93,7 +93,7 @@ struct user_pt_regs { }; struct user_fpsimd_state { - __uint128_t vregs[32]; + __u128 vregs[32]; __u32 fpsr; __u32 fpcr; __u32 __reserved[2]; @@ -258,14 +258,14 @@ struct user_pac_mask { /* pointer authentication keys (NT_ARM_PACA_KEYS, NT_ARM_PACG_KEYS) */ struct user_pac_address_keys { - __uint128_t apiakey; - __uint128_t apibkey; - __uint128_t apdakey; - __uint128_t apdbkey; + __u128 apiakey; + __u128 apibkey; + __u128 apdakey; + __u128 apdbkey; }; struct user_pac_generic_keys { - __uint128_t apgakey; + __u128 apgakey; }; /* ZA state (NT_ARM_ZA) */ diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h index e29bf3e2d0cc..d250ca7a1d46 100644 --- a/arch/arm64/include/uapi/asm/sigcontext.h +++ b/arch/arm64/include/uapi/asm/sigcontext.h @@ -78,7 +78,7 @@ struct fpsimd_context { struct _aarch64_ctx head; __u32 fpsr; __u32 fpcr; - __uint128_t vregs[32]; + __u128 vregs[32]; }; /* @@ -266,8 +266,8 @@ struct gcs_context { * - ---- ----------- * REGS the entire SVE context * - * ZREGS __uint128_t[SVE_NUM_ZREGS][vq] all Z-registers - * ZREG __uint128_t[vq] individual Z-register Zn + * ZREGS __u128[SVE_NUM_ZREGS][vq] all Z-registers + * ZREG __u128[vq] individual Z-register Zn * * PREGS uint16_t[SVE_NUM_PREGS][vq] all P-registers * PREG uint16_t[vq] individual P-register Pn diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648..b5ba9c455aef 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -309,7 +309,7 @@ static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *_ sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56)); } -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009 { @@ -733,10 +733,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .match_list = qcom_erratum_1003_list, }, #endif -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC { .desc = "Broken broadcast TLBI completion", - .capability = ARM64_WORKAROUND_REPEAT_TLBI, + .capability = ARM64_WORKAROUND_REPEAT_TLBI_SYNC, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, .match_list = arm64_repeat_tlbi_list, @@ -850,6 +850,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, #endif +#ifdef CONFIG_NVIDIA_OLYMPUS_1027_ERRATUM + { + /* NVIDIA Olympus core */ + .desc = "NVIDIA Olympus device store/load ordering erratum", + .capability = ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027, + ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE { /* diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9a22df0c5120..4025d51d2d93 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2131,45 +2131,39 @@ static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); } -bool cpu_supports_bbml2_noabort(void) +bool cpu_supports_bbml3(void) { - /* - * We want to allow usage of BBML2 in as wide a range of kernel contexts - * as possible. This list is therefore an allow-list of known-good - * implementations that both support BBML2 and additionally, fulfill the - * extra constraint of never generating TLB conflict aborts when using - * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain - * kernel contexts difficult to prove safe against recursive aborts). - * - * Note that implementations can only be considered "known-good" if their - * implementors attest to the fact that the implementation never raises - * TLB conflict aborts for BBML2 mapping granularity changes. - */ - static const struct midr_range supports_bbml2_noabort_list[] = { + /* CPUs that support BBML3 but dont advertise through ID_AA64MMFR2_EL1 */ + static const struct midr_range supports_bbml3_list[] = { MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3AE, 0, 2, 0xf), MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), MIDR_ALL_VERSIONS(MIDR_AMPERE1), MIDR_ALL_VERSIONS(MIDR_AMPERE1A), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A520AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N3), + MIDR_ALL_VERSIONS(MIDR_C1_NANO), + MIDR_ALL_VERSIONS(MIDR_C1_PRO), + /* Erratum 3683289 fixed in r1p1 */ + MIDR_RANGE(MIDR_C1_ULTRA, 1, 1, 0xf, 0xf), + MIDR_RANGE(MIDR_C1_PREMIUM, 1, 1, 0xf, 0xf), {} }; + u64 mmfr2 = __read_sysreg_by_encoding(SYS_ID_AA64MMFR2_EL1); - /* Does our cpu guarantee to never raise TLB conflict aborts? */ - if (!is_midr_in_range_list(supports_bbml2_noabort_list)) - return false; + if (SYS_FIELD_GET(ID_AA64MMFR2_EL1, BBM, mmfr2) >= ID_AA64MMFR2_EL1_BBM_3) + return true; - /* - * We currently ignore the ID_AA64MMFR2_EL1 register, and only care - * about whether the MIDR check passes. - */ - - return true; + return is_midr_in_range_list(supports_bbml3_list); } -static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope) +static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope) { - return cpu_supports_bbml2_noabort(); + return cpu_supports_bbml3(); } static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) @@ -3062,10 +3056,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) }, { - .desc = "BBM Level 2 without TLB conflict abort", - .capability = ARM64_HAS_BBML2_NOABORT, + .desc = "BBM Level 3", + .capability = ARM64_HAS_BBML3, .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, - .matches = has_bbml2_noabort, + .matches = has_bbml3, }, { .desc = "52-bit Virtual Addressing for KVM (LPA2)", diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 29307642f4c9..e271fbac5f82 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -40,10 +40,7 @@ u8 debug_monitors_arch(void) */ static void mdscr_write(u64 mdscr) { - unsigned long flags; - flags = local_daif_save(); write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); } NOKPROBE_SYMBOL(mdscr_write); diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f804398..0ec90fd1754e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a..72c03ccea59f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -495,6 +515,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, state = irqentry_nmi_enter(regs); do_interrupt_handler(regs, handler); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -509,7 +530,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) @@ -540,6 +561,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) local_daif_restore(DAIF_ERRCTX); state = irqentry_nmi_enter(regs); do_serror(regs, esr); + local_daif_mask(); irqentry_nmi_exit(regs, state); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e0db14e9c843..f63049ac32dc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -333,10 +333,6 @@ alternative_else_nop_endif .endm .macro kernel_exit, el - .if \el != 0 - disable_daif - .endif - #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 5a1554a44162..e1a3c0b3a051 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -409,7 +409,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ftrace_modify_code(pc, old, new, true); } -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \ + defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { @@ -417,7 +418,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, u32 old, new; int ret; - ret = ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec)); + ret = ftrace_rec_update_ops(rec); if (ret) return ret; diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba..7bf117427777 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 05cfb347ec26..42c5543f9589 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -11,6 +11,29 @@ #include #include +enum { + ARM64_IDLE_WFI, + ARM64_IDLE_YIELD, + ARM64_IDLE_NOP, +} idle = ARM64_IDLE_WFI; + +static int __init setup_idle(char *arg) +{ + if (!arg) + return -1; + else if (!strcmp(arg, "wfi")) + idle = ARM64_IDLE_WFI; + else if (!strcmp(arg, "yield")) + idle = ARM64_IDLE_YIELD; + else if (!strcmp(arg, "nop")) + idle = ARM64_IDLE_NOP; + else + return -1; + + return 0; +} +early_param("idle", setup_idle); + /* * cpu_do_idle() * @@ -26,8 +49,13 @@ void __cpuidle cpu_do_idle(void) arm_cpuidle_save_irq_context(&context); - dsb(sy); - wfi(); + if (likely(idle == ARM64_IDLE_WFI)) { + dsb(sy); + wfi(); + } else if (idle == ARM64_IDLE_YIELD) { + dsb(sy); + asm volatile("yield" ::: "memory"); + } arm_cpuidle_restore_irq_context(&context); } diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 7bb6553fec08..3bcf86154d95 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -1023,6 +1023,11 @@ static int __init parse_spectre_bhb_param(char *str) } early_param("nospectre_bhb", parse_spectre_bhb_param); +static bool spectre_bhb_mitigations_off(void) +{ + return __nospectre_bhb || cpu_mitigations_off(); +} + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1036,6 +1041,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { /* Do nothing */ + } else if (spectre_bhb_mitigations_off()) { + /* Mitigation disabled on the command line */ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); @@ -1201,6 +1208,6 @@ void spectre_print_disabled_mitigations(void) if (spectre_v4_mitigations_off()) pr_info("spectre-v4 %s", spectre_disabled_suffix); - if (__nospectre_bhb || cpu_mitigations_off()) + if (spectre_bhb_mitigations_off()) pr_info("spectre-bhb %s", spectre_disabled_suffix); } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 4d08598e2891..4955166723b8 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target, return membuf_write(&to, uregs, sizeof(*uregs)); } +static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) +{ + struct pt_regs *regs = task_pt_regs(target); + struct kernel_siginfo *info = target->last_siginfo; + + /* + * Skip the update for NO_SYSCALL (set either by the user or the + * tracer), as regs[0] holds the return value (see the comment in + * el0_svc_common()) and can be unwound using syscall_rollback(). + */ + if (regs->syscallno == NO_SYSCALL) + return; + + /* We should only be called when target is in a ptrace stop */ + if (WARN_ON_ONCE(!info)) + return; + + /* + * For compat tasks, orig_r0 is provided directly through GPR index + * 17. + */ + if (is_compat_thread(task_thread_info(target))) + return; + + /* + * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the + * return value of the system call. + */ + if ((info->si_code & ~0x80) == SIGTRAP && + target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { + return; + } + + regs->orig_x0 = regs->regs[0]; +} + static int gpr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) @@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, return -EINVAL; task_pt_regs(target)->user_regs = newregs; + + /* + * Keep orig_x0 authoritative so that seccomp (via + * syscall_get_arguments()), audit and the restart path all see the same + * first argument the syscall is dispatched with, even if it has been + * updated by a tracer. + */ + update_syscall_orig_x0_after_ptrace(target); return 0; } @@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target, return ret; task_pt_regs(target)->syscallno = syscallno; + + /* + * Re-sync orig_x0 in case the syscall number has been changed + * from NO_SYSCALL. + */ + update_syscall_orig_x0_after_ptrace(target); return ret; } @@ -801,7 +851,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +887,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index 92160f2e57ff..25ca75ce1a4d 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -23,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared); DEFINE_STATIC_KEY_FALSE_RO(rsi_present); EXPORT_SYMBOL(rsi_present); -bool cc_platform_has(enum cc_attr attr) -{ - switch (attr) { - case CC_ATTR_MEM_ENCRYPT: - return is_realm_world(); - default: - return false; - } -} -EXPORT_SYMBOL_GPL(cc_platform_has); - static bool rsi_version_matches(void) { unsigned long ver_lower, ver_higher; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index cdcdd160e5b6..a61dc3016a11 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -866,14 +868,62 @@ void arch_irq_work_raise(void) } #endif -static void __noreturn local_cpu_stop(unsigned int cpu) +/** + * arm64_nmi_cpu_stop() - stop the local CPU after it is told to stop. + * @regs: register state to record in the vmcore on a crash stop, or NULL for + * panic_smp_self_stop(), which has no interrupted context to save. + * @die_on_crash: on the kdump crash path, power the CPU off via PSCI CPU_OFF + * (so a capture kernel can reclaim it) rather than parking it. + * + * The single point every arm64 stop path funnels through, keeping the + * bookkeeping (mask interrupts, save the crash context, mark offline, mask + * SDEI, optionally power off) in one place: + * + * - the regular IPI_CPU_STOP and pseudo-NMI IPI_CPU_STOP_NMI handlers; + * - panic_smp_self_stop(), a CPU parking itself on a parallel panic(); + * - the SDEI cross-CPU NMI handler (drivers/firmware/arm_sdei_nmi.c), + * which reaches CPUs the stop IPIs could not. + * + * The IPI stop handlers pass @die_on_crash true. The SDEI handler and + * panic_smp_self_stop() pass false and only park. For SDEI that is required, + * not just conservative: it runs inside an SDEI event that is deliberately + * never completed (completing it has firmware resume the wedged context), and + * a CPU_OFF from that not-yet-completed context wedges EL3 on some firmware -- + * a documented follow-up. Parking also matches this path's own fallback when + * CPU_OFF is unavailable. + */ +void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash) { + unsigned int cpu = smp_processor_id(); + bool crash = IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop; + + /* + * Use local_daif_mask() instead of local_irq_disable() to make sure + * that pseudo-NMIs are disabled. The "stop" code starts with an IRQ + * and falls back to NMI (which might be pseudo). If the IRQ finally + * goes through right as we're timing out then the NMI could interrupt + * us. It's better to prevent the NMI and let the IRQ finish since the + * pt_regs will be better. + */ + local_daif_mask(); + +#ifdef CONFIG_KEXEC_CORE + if (crash && regs) + crash_save_cpu(regs, cpu); +#endif + + /* the ack a stop requester (e.g. smp_send_stop()) polls for */ set_cpu_online(cpu, false); - local_daif_mask(); sdei_mask_local_cpu(); + + if (crash && die_on_crash) + __cpu_try_die(cpu); + + /* just in case */ cpu_park_loop(); } +NOKPROBE_SYMBOL(arm64_nmi_cpu_stop); /* * We need to implement panic_smp_self_stop() for parallel panic() calls, so @@ -882,36 +932,7 @@ static void __noreturn local_cpu_stop(unsigned int cpu) */ void __noreturn panic_smp_self_stop(void) { - local_cpu_stop(smp_processor_id()); -} - -static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) -{ -#ifdef CONFIG_KEXEC_CORE - /* - * Use local_daif_mask() instead of local_irq_disable() to make sure - * that pseudo-NMIs are disabled. The "crash stop" code starts with - * an IRQ and falls back to NMI (which might be pseudo). If the IRQ - * finally goes through right as we're timing out then the NMI could - * interrupt us. It's better to prevent the NMI and let the IRQ - * finish since the pt_regs will be better. - */ - local_daif_mask(); - - crash_save_cpu(regs, cpu); - - set_cpu_online(cpu, false); - - sdei_mask_local_cpu(); - - if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) - __cpu_try_die(cpu); - - /* just in case */ - cpu_park_loop(); -#else - BUG(); -#endif + arm64_nmi_cpu_stop(NULL, false); } static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr) @@ -932,6 +953,16 @@ static void arm64_backtrace_ipi(cpumask_t *mask) void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { + /* + * Prefer the SDEI cross-CPU NMI provider when active: firmware + * dispatches the event out of EL3 and reaches CPUs that have + * interrupts locally masked, without the per-IRQ-mask cost that + * pseudo-NMI pays for the same reach. The plain IPI path below + * can't reach such a CPU unless pseudo-NMI is enabled. + */ + if (sdei_nmi_trigger_cpumask_backtrace(mask, exclude_cpu)) + return; + /* * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name, * nothing about it truly needs to be implemented using an NMI, it's @@ -978,12 +1009,7 @@ static void do_handle_IPI(int ipinr) case IPI_CPU_STOP: case IPI_CPU_STOP_NMI: - if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) { - ipi_cpu_crash_stop(cpu, get_irq_regs()); - unreachable(); - } else { - local_cpu_stop(cpu); - } + arm64_nmi_cpu_stop(get_irq_regs(), true); break; #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST @@ -1086,7 +1112,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } @@ -1257,6 +1283,28 @@ void smp_send_stop(void) udelay(1); } + /* + * If CPUs are *still* online, try the SDEI cross-CPU NMI. Firmware + * delivers it regardless of the target's DAIF state, so it reaches + * a CPU spinning with interrupts masked, which neither rung above + * could (without pseudo-NMI there is no NMI rung at all). Allow + * 100ms: a firmware round-trip per CPU, with headroom. + */ + if (num_other_online_cpus() && sdei_nmi_active()) { + /* re-snapshot after the rungs above took CPUs offline */ + smp_rmb(); + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + pr_info("SMP: retry stop with SDEI NMI for CPUs %*pbl\n", + cpumask_pr_args(&mask)); + + sdei_nmi_stop_cpus(&mask); + timeout = USEC_PER_MSEC * 100; + while (num_other_online_cpus() && timeout--) + udelay(1); + } + if (num_other_online_cpus()) { smp_rmb(); cpumask_copy(&mask, cpu_online_mask); diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index eaaff94329cd..c41724a40b75 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; - struct arm_cpuidle_irq_context context; /* * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized @@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * Strictly speaking the trace_hardirqs_off() here is superfluous, * hardirqs should be firmly off by now. This really ought to use * something like raw_local_daif_save(). + * + * This also unmasks interrupts in PMR in order to reliably + * resume if we're using pseudo-NMIs. */ flags = local_daif_save(); @@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ pause_graph_tracing(); - /* - * Switch to using DAIF.IF instead of PMR in order to reliably - * resume if we're using pseudo-NMIs. - */ - arm_cpuidle_save_irq_context(&context); - ct_cpuidle_enter(); if (__cpu_suspend_enter(&state)) { @@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) __cpu_suspend_exit(); } - arm_cpuidle_restore_irq_context(&context); - unpause_graph_tracing(); /* diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index b32f13358fbb..d28438f8b83f 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -373,6 +373,16 @@ core_initcall(init_amu_fie); #ifdef CONFIG_ACPI_CPPC_LIB #include +struct amu_ffh_ctrs { + u64 corecnt; + u64 constcnt; +}; + +enum cpc_ffh_ctr_id { + CPC_FFH_CTR_CORE = 0x0, + CPC_FFH_CTR_CONST = 0x1, +}; + static void cpu_read_corecnt(void *val) { /* @@ -397,7 +407,7 @@ static void cpu_read_constcnt(void *val) } static inline -int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) +int counters_read_on_cpu(int cpu, smp_call_func_t func, void *val) { /* * Abort call on counterless CPU. @@ -447,24 +457,90 @@ bool cpc_ffh_supported(void) return true; } +static void amu_read_core_const_ctrs(void *val) +{ + struct amu_ffh_ctrs *ctrs = val; + + /* + * cpu_read_constcnt() incurs slight latency due to the + * ARM64_WORKAROUND_2457168 check. Read it first to minimize + * the sampling skew between the const and core counters. + */ + cpu_read_constcnt(&ctrs->constcnt); + cpu_read_corecnt(&ctrs->corecnt); +} + +static u64 cpc_ffh_extract_bits(const struct cpc_reg *reg, u64 val) +{ + val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, + reg->bit_offset); + val >>= reg->bit_offset; + + return val; +} + +static void cpc_ffh_ctr_value(const struct cpc_reg *reg, + const struct amu_ffh_ctrs *ctrs, u64 *val) +{ + switch ((u64)reg->address) { + case CPC_FFH_CTR_CORE: + *val = ctrs->corecnt; + break; + case CPC_FFH_CTR_CONST: + *val = ctrs->constcnt; + break; + } + + *val = cpc_ffh_extract_bits(reg, *val); +} + +static bool is_amu_ctr_reg(const struct cpc_reg *reg) +{ + return reg->address == CPC_FFH_CTR_CORE || + reg->address == CPC_FFH_CTR_CONST; +} + +int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + struct amu_ffh_ctrs ctrs; + int ret; + + if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2)) + return -EINVAL; + + ret = counters_read_on_cpu(cpu, amu_read_core_const_ctrs, &ctrs); + if (ret) { + /* + * If AMU is unsupported (-EOPNOTSUPP), translate the error + * to -ENODEV. This explicitly tells the generic CPPC layer + * to abort immediately and avoid falling back to pointless + * single-counter reads. + */ + return ret == -EOPNOTSUPP ? -ENODEV : ret; + } + + cpc_ffh_ctr_value(reg1, &ctrs, val1); + cpc_ffh_ctr_value(reg2, &ctrs, val2); + + return 0; +} + int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) { int ret = -EOPNOTSUPP; switch ((u64)reg->address) { - case 0x0: + case CPC_FFH_CTR_CORE: ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); break; - case 0x1: + case CPC_FFH_CTR_CONST: ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); break; } - if (!ret) { - *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, - reg->bit_offset); - *val >>= reg->bit_offset; - } + if (!ret) + *val = cpc_ffh_extract_bits(reg, *val); return ret; } diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 448c917494f3..b33e1ca4a781 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 + +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8..0acab179fc1a 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -89,7 +89,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, } /* - * On eliding the __tlb_flush_range() under BBML2+noabort: + * On eliding the __tlb_flush_range() under BBML3: * * NOTE: Instead of using N=16 as the contiguous block length, we use * N=4 for clarity. @@ -135,7 +135,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * contiguous TLB entry, which is a micro-optimisation opportunity, * but does not affect correctness. * - * In the BBML2 case, the change is avoiding the intermediate tlbi+dsb. + * In the BBML3 case, the change is avoiding the intermediate tlbi+dsb. * This means a few things, but notably other PEs will still "see" any * stale cached TLB entries. This could lead to a "contiguous bit * misprogramming" issue until the final tlbi+dsb of the changed page, @@ -158,21 +158,16 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * are present, and a write is made to this address, do we fault or * is the write permitted (via amalgamation)? * - * The relevant Arm ARM DDI 0487L.a requirements are RNGLXZ and RJQQTC, - * and together state that when BBML1 or BBML2 are implemented, either - * a TLB conflict abort is raised (which we expressly forbid), or will - * "produce an OA, access permissions, and memory attributes that are - * consistent with any of the programmed translation table values". - * - * That is to say, will either raise a TLB conflict, or produce one of - * the cached TLB entries, but never amalgamate. + * With BBML3 implemented, no TLB conflict abort is raised and the OA, + * access permissions and memory attributes produced is one of the cached + * TLB entries, but never amalgamate. * * Thus, as the page tables are only considered "consistent" after * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB * entry as well as the new contiguous (RO,c) TLB entry), omitting the * initial tlbi+dsb is correct. * - * It is also important to note that at the end of the BBML2 folding + * It is also important to note that at the end of the BBML3 folding * case, we are still left with potentially all N TLB entries still * cached (the N-1 non-contiguous ptes, and the single contiguous * block). However, over time, natural TLB pressure will cause the @@ -214,7 +209,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * * |____| <--- tlbi + dsb * - * For BBML2, we again remove the intermediate tlbi+dsb. Here, there + * For BBML3, we again remove the intermediate tlbi+dsb. Here, there * are no issues, as the final tlbi+dsb covering the changed page is * guaranteed to remove the original large contiguous (RW,c) TLB entry, * as well as the intermediate (RW,n) TLB entry; the next access will @@ -224,7 +219,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * regardless. */ - if (!system_supports_bbml2_noabort()) + if (!system_supports_bbml3()) __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, TLBF_NOWALKCACHE); diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 994b7b36e2b9..7f6f2f40dba2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -42,6 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); + if (!coherent && !CLIDR_LOC(read_sysreg(clidr_el1))) { + dev_warn(dev, "CLIDR_EL1.LoC == 0, treating as coherent\n"); + coherent = true; + } + WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN, TAINT_CPU_OUT_OF_SPEC, "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)", diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 227f58522dad..52282d561a0d 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include +#include #include #include #include @@ -337,25 +339,21 @@ void __init arch_setup_zero_pages(void) void __init arch_mm_preinit(void) { unsigned int flags = SWIOTLB_VERBOSE; - bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit); - if (is_realm_world()) { - swiotlb = true; + if (is_realm_world() || is_protected_kvm_guest()) { flags |= SWIOTLB_FORCE; - } - - if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) { + } else if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) { /* * If no bouncing needed for ZONE_DMA, reduce the swiotlb * buffer for kmalloc() bouncing to 1MB per 1GB of RAM. */ unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); + swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); - swiotlb = true; } - swiotlb_init(swiotlb, flags); + swiotlb_init(true, flags); /* * Check boundaries twice: Some fundamental inconsistencies can be @@ -417,6 +415,17 @@ void dump_mem_limit(void) } } +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + return is_realm_world() || is_protected_kvm_guest(); + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + #ifdef CONFIG_EXECMEM static u64 module_direct_base __ro_after_init = 0; static u64 module_plt_base __ro_after_init = 0; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d4de88770ecf..79d90226fd5d 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -777,18 +778,18 @@ static int split_kernel_leaf_mapping_locked(unsigned long addr) static inline bool force_pte_mapping(void) { - const bool bbml2 = system_capabilities_finalized() ? - system_supports_bbml2_noabort() : cpu_supports_bbml2_noabort(); + const bool bbml3 = system_capabilities_finalized() ? + system_supports_bbml3() : cpu_supports_bbml3(); if (debug_pagealloc_enabled()) return true; - if (bbml2) + if (bbml3) return false; return rodata_full || arm64_kfence_can_set_direct_map() || is_realm_world(); } static DEFINE_MUTEX(pgtable_split_lock); -static bool linear_map_requires_bbml2; +static bool linear_map_requires_bbml3; int split_kernel_leaf_mapping(unsigned long start, unsigned long end) { @@ -801,15 +802,15 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) * always pte-mapped), we must not go any further because taking the * mutex below may sleep. Do not call force_pte_mapping() here because * it could return a confusing result if called from a secondary cpu - * prior to finalizing caps. Instead, linear_map_requires_bbml2 gives us + * prior to finalizing caps. Instead, linear_map_requires_bbml3 gives us * what we need. */ - if (!linear_map_requires_bbml2 || is_kfence_address((void *)start)) + if (!linear_map_requires_bbml3 || is_kfence_address((void *)start)) return 0; - if (!system_supports_bbml2_noabort()) { + if (!system_supports_bbml3()) { /* - * !BBML2_NOABORT systems should not be trying to change + * BBML3 systems should not be trying to change * permissions on anything that is not pte-mapped in the first * place. Just return early and let the permission change code * raise a warning if not already pte-mapped. @@ -826,8 +827,8 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) /* * Boot-time: Started secondary cpus but don't know if they - * support BBML2_NOABORT yet. Can't allow splitting in this - * window in case they don't. + * support BBML3 yet. Can't allow splitting in this window + * in case they don't. */ if (WARN_ON(num_online_cpus() > 1)) return -EBUSY; @@ -932,11 +933,11 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp return ret; } -u32 idmap_kpti_bbml2_flag; +u32 idmap_kpti_bbml3_flag; -static void __init init_idmap_kpti_bbml2_flag(void) +static void __init init_idmap_kpti_bbml3_flag(void) { - WRITE_ONCE(idmap_kpti_bbml2_flag, 1); + WRITE_ONCE(idmap_kpti_bbml3_flag, 1); /* Must be visible to other CPUs before stop_machine() is called. */ smp_mb(); } @@ -945,7 +946,7 @@ static int __init linear_map_split_to_ptes(void *__unused) { /* * Repainting the linear map must be done by CPU0 (the boot CPU) because - * that's the only CPU that we know supports BBML2. The other CPUs will + * that's the only CPU that we know supports BBML3. The other CPUs will * be held in a waiting area with the idmap active. */ if (!smp_processor_id()) { @@ -958,7 +959,7 @@ static int __init linear_map_split_to_ptes(void *__unused) /* * Wait for all secondary CPUs to be put into the waiting area. */ - smp_cond_load_acquire(&idmap_kpti_bbml2_flag, VAL == num_online_cpus()); + smp_cond_load_acquire(&idmap_kpti_bbml3_flag, VAL == num_online_cpus()); /* * Walk all of the linear map [lstart, lend), except the kernel @@ -977,7 +978,7 @@ static int __init linear_map_split_to_ptes(void *__unused) * Relies on dsb in flush_tlb_kernel_range() to avoid reordering * before any page table split operations. */ - WRITE_ONCE(idmap_kpti_bbml2_flag, 0); + WRITE_ONCE(idmap_kpti_bbml3_flag, 0); } else { typedef void (wait_split_fn)(void); extern wait_split_fn wait_linear_map_split_to_ptes; @@ -986,7 +987,7 @@ static int __init linear_map_split_to_ptes(void *__unused) wait_fn = (void *)__pa_symbol(wait_linear_map_split_to_ptes); /* - * At least one secondary CPU doesn't support BBML2 so cannot + * At least one secondary CPU doesn't support BBML3 so cannot * tolerate the size of the live mappings changing. So have the * secondary CPUs wait for the boot CPU to make the changes * with the idmap active and init_mm inactive. @@ -1001,8 +1002,8 @@ static int __init linear_map_split_to_ptes(void *__unused) void __init linear_map_maybe_split_to_ptes(void) { - if (linear_map_requires_bbml2 && !system_supports_bbml2_noabort()) { - init_idmap_kpti_bbml2_flag(); + if (linear_map_requires_bbml3 && !system_supports_bbml3()) { + init_idmap_kpti_bbml3_flag(); stop_machine(linear_map_split_to_ptes, NULL, cpu_online_mask); } } @@ -1060,6 +1061,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1068,6 +1092,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read/write before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION) && rodata_enabled) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1125,7 +1164,7 @@ bool arch_kfence_init_pool(void) mutex_unlock(&pgtable_split_lock); /* - * Since the system supports bbml2_noabort, tlb invalidation is not + * Since the system supports bbml3, tlb invalidation is not * required here; the pgtable mappings have been split to pte but larger * entries may safely linger in the TLB. */ @@ -1164,7 +1203,7 @@ static void __init map_mem(void) arm64_kfence_map_pool(); - linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); + linear_map_requires_bbml3 = !force_pte_mapping() && can_set_direct_map(); if (force_pte_mapping()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; @@ -1188,6 +1227,20 @@ static void __init map_mem(void) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { + /* + * for_each_mem_range may return sub-page-aligned boundaries + * after memblock_mark_nomap() splits regions at byte precision. + * __create_pgd_mapping_locked aligns phys down to PAGE_MASK, + * which could accidentally map no-map memory on the boundary. + * Round the mappable range inward: start UP, end DOWN, so + * that the mapped area never overlaps with adjacent no-map + * regions. The cost is at most one page of unmapped gap at + * each boundary. + */ + start = PAGE_ALIGN(start); + end = end & PAGE_MASK; + if (start >= end) + continue; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as @@ -1215,11 +1268,8 @@ void mark_rodata_ro(void) (unsigned long)_stext - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss read-only in the linear map */ - update_mapping_prot(__pa_symbol(__init_end), - (unsigned long)lm_alias(__init_end), - (unsigned long)__bss_stop - (unsigned long)__init_end, - PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } static void __init declare_vma(struct vm_struct *vma, @@ -1331,7 +1381,7 @@ void __init kpti_install_ng_mappings(void) if (arm64_use_ng_mappings) return; - init_idmap_kpti_bbml2_flag(); + init_idmap_kpti_bbml3_flag(); stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask); } @@ -1392,7 +1442,7 @@ void __pi_map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset); static u8 idmap_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init, - kpti_bbml2_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init; + kpti_bbml3_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init; static void __init create_idmap(void) { @@ -1404,17 +1454,17 @@ static void __init create_idmap(void) IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false, __phys_to_virt(ptep) - ptep); - if (linear_map_requires_bbml2 || + if (linear_map_requires_bbml3 || (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && !arm64_use_ng_mappings)) { - phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml2_flag); + phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml3_flag); /* * The KPTI G-to-nG conversion code needs a read-write mapping * of its synchronization flag in the ID map. This is also used * when splitting the linear map to ptes if a secondary CPU - * doesn't support bbml2. + * doesn't support bbml3. */ - ptep = __pa_symbol(kpti_bbml2_ptes); + ptep = __pa_symbol(kpti_bbml3_ptes); __pi_map_range(&ptep, pa, pa + sizeof(u32), pa, PAGE_KERNEL, IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false, __phys_to_virt(ptep) - ptep); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be37..12aacc74f764 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -169,6 +169,13 @@ alternative_if ARM64_HAS_RAS_EXTN msr_s SYS_DISR_EL1, xzr alternative_else_nop_endif +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if ARM64_HAS_GIC_PRIO_MASKING + mov x1, #GIC_PRIO_IRQON + msr_s SYS_ICC_PMR_EL1, x1 +alternative_else_nop_endif +#endif + ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 isb ret @@ -287,7 +294,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) mov x5, x3 // preserve temp_pte arg mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag cbnz cpu, __idmap_kpti_secondary @@ -445,7 +452,7 @@ SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) flag_ptr .req x4 mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag __idmap_cpu_set_reserved_ttbr1 x16, x17 scondary_cpu_wait: diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..0ee42b0cc8d3 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # -# Internal CPU capabilities constants, keep this list sorted +# Internal CPU capabilities constants ALWAYS_BOOT ALWAYS_SYSTEM @@ -14,6 +14,7 @@ HAS_ADDRESS_AUTH_ARCH_QARMA5 HAS_ADDRESS_AUTH_IMP_DEF HAS_AMU_EXTN HAS_ARMv8_4_TTL +HAS_BBML3 HAS_CACHE_DIC HAS_CACHE_IDC HAS_CNP @@ -51,7 +52,6 @@ HAS_LS64_V HAS_LSUI HAS_MOPS HAS_NESTED_VIRT -HAS_BBML2_NOABORT HAS_PAN HAS_PMUV3 HAS_S1PIE @@ -121,10 +121,11 @@ WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE WORKAROUND_DISABLE_CNP +WORKAROUND_NVIDIA_OLYMPUS_1027 WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF -WORKAROUND_REPEAT_TLBI +WORKAROUND_REPEAT_TLBI_SYNC WORKAROUND_SPECULATIVE_AT WORKAROUND_SPECULATIVE_SSBS WORKAROUND_SPECULATIVE_UNPRIV_LOAD diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 7cb61aca3797..e070ada06196 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2259,6 +2259,7 @@ UnsignedEnum 55:52 BBM 0b0000 0 0b0001 1 0b0010 2 + 0b0011 3 EndEnum UnsignedEnum 51:48 TTL 0b0000 NI diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c index bb010f6164e5..91fcdd289e63 100644 --- a/drivers/acpi/arm64/apmt.c +++ b/drivers/acpi/arm64/apmt.c @@ -76,10 +76,12 @@ static int __init apmt_add_platform_device(struct acpi_apmt_node *node, struct fwnode_handle *fwnode) { struct platform_device *pdev; - int ret, count; + int ret, count, uid = node->id & INT_MAX; struct resource res[DEV_MAX_RESOURCE_COUNT]; - pdev = platform_device_alloc(DEV_NAME, PLATFORM_DEVID_AUTO); + if (uid != node->id) + pr_warn("Unexpectedly large UID 0x%x, truncated to 0x%x\n", node->id, uid); + pdev = platform_device_alloc(DEV_NAME, uid); if (!pdev) return -ENOMEM; diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 2f7c09552566..fef54fcd00b7 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1083,6 +1083,22 @@ int __weak cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) return -ENOTSUPP; } +/** + * cpc_read_ffh_fb_ctrs() - Read FFH feedback counters together + * @cpunum: Target CPU + * @reg1: first CPPC register information + * @val1: place holder for first return value + * @reg2: second CPPC register information + * @val2: place holder for second return value + * + * Return: 0 on success, error code otherwise + */ +int __weak cpc_read_ffh_fb_ctrs(int cpunum, struct cpc_reg *reg1, + u64 *val1, struct cpc_reg *reg2, u64 *val2) +{ + return -EOPNOTSUPP; +} + /** * cpc_write_ffh() - Write FFH register * @cpunum: CPU number to write @@ -1603,6 +1619,33 @@ bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu) } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu); +static int cppc_read_fb_ctrs(int cpunum, + struct cpc_register_resource *delivered_reg, + struct cpc_register_resource *reference_reg, + u64 *delivered, u64 *reference) +{ + int ret; + + /* + * For FFH feedback counters, try a paired read first to reduce + * sampling skew between delivered and reference counters. Fall + * back to the existing per-register reads if unsupported. + */ + if (CPC_IN_FFH(delivered_reg) && CPC_IN_FFH(reference_reg)) { + ret = cpc_read_ffh_fb_ctrs(cpunum, + &delivered_reg->cpc_entry.reg, delivered, + &reference_reg->cpc_entry.reg, reference); + if (ret != -EOPNOTSUPP) + return ret; + } + + ret = cpc_read(cpunum, delivered_reg, delivered); + if (ret) + return ret; + + return cpc_read(cpunum, reference_reg, reference); +} + /** * cppc_perf_ctrs_in_pcc - Check if any perf counters are in a PCC region. * @@ -1668,11 +1711,8 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) } } - ret = cpc_read(cpunum, delivered_reg, &delivered); - if (ret) - goto out_err; - - ret = cpc_read(cpunum, reference_reg, &reference); + ret = cppc_read_fb_ctrs(cpunum, delivered_reg, reference_reg, + &delivered, &reference); if (ret) goto out_err; diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 12dc70254842..dbaca140a1b0 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -36,6 +36,27 @@ config ARM_SDE_INTERFACE standard for registering callbacks from the platform firmware into the OS. This is typically used to implement RAS notifications. +config ARM_SDEI_NMI + bool "SDEI-based cross-CPU NMI service (arm64)" + depends on ARM_SDE_INTERFACE + help + Provides SDEI-based cross-CPU NMI delivery for hooks that need + to reach interrupt-masked CPUs on silicon that lacks FEAT_NMI: + + - arch_trigger_cpumask_backtrace() (sysrq-l, RCU stalls, + hardlockup_all_cpu_backtrace, soft-lockup secondary dumps, + hung-task auxiliary dumps) + - smp_send_stop() escalation (reboot/halt and the + panic / kdump crash stop) + + The driver registers a handler for the SDEI software-signalled + event (event 0) and reaches a target CPU by signalling it with + SDEI_EVENT_SIGNAL. Firmware delivers the event out of EL3 + regardless of the target's PSTATE.DAIF -- forced delivery into a + CPU wedged with interrupts locally masked. + + If unsure, say N. + config EDD tristate "BIOS Enhanced Disk Drive calls determine boot disk" depends on X86 diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 4ddec2820c96..be46f1e1dc77 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_ARM_SCPI_PROTOCOL) += arm_scpi.o obj-$(CONFIG_ARM_SDE_INTERFACE) += arm_sdei.o +obj-$(CONFIG_ARM_SDEI_NMI) += arm_sdei_nmi.o obj-$(CONFIG_DMI) += dmi_scan.o obj-$(CONFIG_DMI_SYSFS) += dmi-sysfs.o obj-$(CONFIG_EDD) += edd.o diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index f39ed7ba3a38..e8dd2f0f3919 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -339,6 +339,28 @@ static void _ipi_unmask_cpu(void *ignored) sdei_unmask_local_cpu(); } +/* + * Signal the software-signalled event (event 0) to @mpidr. Does nothing + * but the SMC -- no locks, no event lookup -- so it is safe from NMI / + * crash context (e.g. the cross-CPU NMI service). + */ +int sdei_event_signal(u32 event_num, u64 mpidr) +{ + return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_SIGNAL, event_num, + mpidr, 0, 0, 0, NULL); +} +NOKPROBE_SYMBOL(sdei_event_signal); + +/* + * Was SDEI firmware probed and is it usable? Lets optional consumers skip + * registering an event -- and the warning a failed registration emits -- on + * systems with no SDEI. + */ +bool sdei_is_present(void) +{ + return sdei_firmware_call; +} + static void _ipi_private_reset(void *ignored) { int err; diff --git a/drivers/firmware/arm_sdei_nmi.c b/drivers/firmware/arm_sdei_nmi.c new file mode 100644 index 000000000000..038eb79e28ac --- /dev/null +++ b/drivers/firmware/arm_sdei_nmi.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm64 SDEI-based cross-CPU NMI service. + * + * Delivering an "NMI-shaped" event to an EL1 context that has locally + * masked interrupts, on silicon without FEAT_NMI, can be done two ways: + * + * - pseudo-NMI: mask "interrupts" via the GIC priority register + * (ICC_PMR_EL1) instead of PSTATE.DAIF, leaving a high-priority band + * deliverable. Functionally this works -- but it reimplements every + * local_irq_disable()/enable() and exception entry/exit as a PMR + * write plus synchronisation, a cost paid on that hot path forever, + * whether or not an NMI is ever delivered. + * + * - SDEI: leave interrupt masking as the cheap PSTATE.DAIF operation + * and have the firmware bounce an EL3-routed Group-0 SGI back to + * NS-EL1 as an event callback. The cost is a firmware round-trip, + * but only at the rare moment delivery is actually needed. + * + * This driver takes the second path: it keeps the IRQ-mask hot path + * free and pays only when it fires, which is what makes cross-CPU NMI + * affordable on hardware where the pseudo-NMI tax isn't, until FEAT_NMI + * makes NMI masking cheap in the architecture itself. + * + * Capabilities provided: + * + * - sdei_nmi_trigger_cpumask_backtrace() — override for arm64's + * arch_trigger_cpumask_backtrace(), so sysrq-l, RCU stall dumps, + * hardlockup_all_cpu_backtrace, soft-lockup/hung-task secondary + * dumps all reach interrupt-masked CPUs. + * + * - sdei_nmi_stop_cpus() — the last rung of smp_send_stop()'s + * escalation (reboot/halt and the panic/kdump crash stop alike), + * reaching CPUs that ignored the stop IPIs; on the kdump path the + * wedged context is captured into the vmcore before the CPU parks. + * + * Delivery uses the standard SDEI software-signalled event (event 0) and + * SDEI_EVENT_SIGNAL. We register a handler for event 0, enable it, and + * poke a target CPU with sdei_event_signal(0, mpidr): firmware makes + * event 0 pending on that PE and dispatches the handler NMI-like, + * regardless of the target's DAIF. + * Availability is simply whether event 0 registers and enables -- if SDEI + * and its software-signalled event are present we use it, otherwise the + * driver stays inert. + */ + +#define pr_fmt(fmt) "sdei_nmi: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static bool sdei_nmi_available; + +#define SDEI_NMI_EVENT 0 + +/* + * Backtrace and stop both ride SDEI event 0. That is not a chosen economy: + * event 0 is the only architecturally software-signalled event -- the sole + * event SDEI_EVENT_SIGNAL can target at an arbitrary PE. Every other event + * number is a firmware/platform interrupt-bound event, not something the + * kernel can raise cross-CPU, so a dedicated "stop" event would need + * firmware to define and bind it -- exactly the firmware dependency this + * driver sets out to avoid. + * + * Sharing one event means the handler must tell a stop apart from a + * backtrace. A stop is terminal and system-wide -- sdei_nmi_stop_cpus() is + * only reached from smp_send_stop() (reboot/halt/panic/kdump), which never + * returns -- so once a stop is requested, every later event-0 fire is a + * stop too. A single write-once flag therefore carries as much as a + * per-CPU mask would: sdei_nmi_stop_cpus() sets it before signalling, and + * the handler reads a set flag as "stop this CPU" and a clear flag as + * "backtrace" (handled by nmi_cpu_backtrace(), which self-gates on the + * framework's backtrace mask). A backtrace fire that races in after a stop + * has begun just stops that CPU instead -- harmless, it is going down. + */ +static bool sdei_nmi_stopping; + +static int sdei_nmi_handler(u32 event, struct pt_regs *regs, void *arg) +{ + /* + * No smp_rmb() pairing sdei_nmi_stop_cpus()'s dsb(ishst): the flag is + * the only shared value, and this handler runs only because firmware + * delivered the event -- a round-trip past that store -- so the read + * cannot be stale and there is no second load for a barrier to order. + */ + if (READ_ONCE(sdei_nmi_stopping)) { + /* + * Never returns, and deliberately never completes the SDEI + * event: SDEI_EVENT_COMPLETE has firmware restore the + * interrupted context, which would land the CPU back in + * the wedged loop (or in do_idle, which BUGs at + * cpuhp_report_idle_dead once it sees itself offline). + * Returning a modified pt_regs doesn't help -- + * arch/arm64/kernel/sdei.c::do_sdei_event only honours a PC + * override via its IRQ-state heuristic and otherwise hands + * EL3 its own saved-context slot back. + * + * Trade-off: EL3 retains ~one saved-context slot per parked + * CPU until the next hardware reset (~hundreds of bytes per + * CPU). Recoverability is unchanged versus an IPI-stopped + * CPU: neither comes back without a reset. + */ + arm64_nmi_cpu_stop(regs, false); + /* unreachable */ + } + + /* + * nmi_cpu_backtrace() no-ops unless this CPU's bit is set in the + * global backtrace mask (driven by nmi_trigger_cpumask_backtrace()), + * so a fire that reaches a CPU not being backtraced is harmless. + */ + nmi_cpu_backtrace(regs); + return SDEI_EV_HANDLED; +} +NOKPROBE_SYMBOL(sdei_nmi_handler); + +static void sdei_nmi_fire(unsigned int target_cpu) +{ + int err = sdei_event_signal(SDEI_NMI_EVENT, cpu_logical_map(target_cpu)); + + if (err) + pr_warn("SDEI_EVENT_SIGNAL to CPU %u failed: %d\n", + target_cpu, err); +} + +/* + * Raise callback for nmi_trigger_cpumask_backtrace(): signal event 0 + * at every CPU still pending in @mask. The framework excludes the local + * CPU from @mask before calling us. + */ +static void sdei_nmi_raise_backtrace(cpumask_t *mask) +{ + unsigned int cpu; + + /* + * Publish backtrace_mask (set by nmi_trigger_cpumask_backtrace()) + * before signalling. As in the stop path, the SMC is not a memory + * store, so dsb(ishst) is needed for the target to observe the mask. + */ + dsb(ishst); + + for_each_cpu(cpu, mask) + sdei_nmi_fire(cpu); +} + +/* + * Override hook for arch_trigger_cpumask_backtrace() (see + * arch/arm64/kernel/smp.c). Returns true when SDEI handled the request, + * which is the case whenever SDEI is active; on a false return the arch + * falls back to its regular-IRQ (or pseudo-NMI, if enabled) IPI. + * + * On a kernel built without paying the pseudo-NMI hot-path cost (the + * usual case for this driver's target), the IPI can't reach a CPU that + * has interrupts masked -- so the backtrace of the one CPU you care + * about comes back empty. SDEI is dispatched out of EL3 and lands + * regardless of the target's DAIF, without taxing the IRQ-mask path. + */ +bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + if (!sdei_nmi_available) + return false; + + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, + sdei_nmi_raise_backtrace); + return true; +} + +bool sdei_nmi_active(void) +{ + return sdei_nmi_available; +} + +/* + * Last rung of the stop escalation in smp_send_stop() (see + * arch/arm64/kernel/smp.c). The caller runs the regular stop IPI (and + * the pseudo-NMI stop IPI, where available) first; @mask holds whatever + * stayed online through those -- typically CPUs wedged with interrupts + * masked, unreachable by an IPI. Mark the stop in progress and signal + * event 0 at each target; a target acks by marking itself offline, which + * the caller polls for. The caller has already confirmed sdei_nmi_active(). + */ +void sdei_nmi_stop_cpus(const cpumask_t *mask) +{ + unsigned int cpu; + + WRITE_ONCE(sdei_nmi_stopping, true); + + /* + * Publish the flag before signalling. The signal goes out via an SMC + * to firmware, not a memory store, so smp_wmb() ordering is not + * enough: use dsb(ishst) to make the store globally visible before the + * SMC executes, as gic_ipi_send_mask() does for its SGI. The SDEI spec + * does not require the dispatch to order the caller's prior stores. + */ + dsb(ishst); + + for_each_cpu(cpu, mask) + sdei_nmi_fire(cpu); +} + +/* + * device_initcall (after arch_initcall(sdei_init), so the SDEI subsystem + * is up): probe the firmware, register the event, and turn on the + * cross-CPU service. If the probe fails the driver stays inert and the + * override hooks decline, leaving the arch's own paths in place. + */ +static int __init sdei_nmi_init(void) +{ + int err; + + if (!sdei_is_present()) + return 0; + + err = sdei_event_register(SDEI_NMI_EVENT, sdei_nmi_handler, NULL); + if (err) { + pr_err("sdei_event_register(%u) failed: %d\n", + SDEI_NMI_EVENT, err); + return 0; + } + + err = sdei_event_enable(SDEI_NMI_EVENT); + if (err) { + pr_err("sdei_event_enable(%u) failed: %d\n", + SDEI_NMI_EVENT, err); + sdei_event_unregister(SDEI_NMI_EVENT); + return 0; + } + + sdei_nmi_available = true; + pr_info("using SDEI cross-CPU NMI (SDEI_EVENT_SIGNAL, event %u)\n", + SDEI_NMI_EVENT); + + return 0; +} +device_initcall(sdei_nmi_init); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 1ed8a6f29dc4..01492fb52659 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -208,7 +208,7 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) feat_mask |= ARM_SMMU_FEAT_VAX; } - if (system_supports_bbml2_noabort()) + if (system_supports_bbml3()) feat_mask |= ARM_SMMU_FEAT_BBML2; if ((smmu->features & feat_mask) != feat_mask) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 2673954d4577..6e1fa5b247fc 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -867,10 +867,7 @@ static void __gic_handle_irq_from_irqson(struct pt_regs *regs) nmi_exit(); } - if (gic_prio_masking_enabled()) { - gic_pmr_mask_irqs(); - gic_arch_enable_irqs(); - } + gic_unmask_pnmis(); if (!is_nmi) __gic_handle_irq(irqnr, regs); diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c index ac49d3b2dad6..de77da83df2e 100644 --- a/drivers/perf/alibaba_uncore_drw_pmu.c +++ b/drivers/perf/alibaba_uncore_drw_pmu.c @@ -449,11 +449,8 @@ static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device */ ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, IRQF_SHARED, dev_name(&pdev->dev), irq); - if (ret < 0) { - dev_err(&pdev->dev, - "Fail to request IRQ:%d ret:%d\n", irq_num, ret); + if (ret < 0) goto out_free; - } ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); if (ret) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 6e5cc4086a9e..a03e2a8ca41e 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -29,9 +29,11 @@ #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) #define CMN_CHILD_NODE_ADDR GENMASK(29, 0) +#define CMN_CHILD_NODE_ISOLATED BIT(30) #define CMN_CHILD_NODE_EXTERNAL BIT(31) -#define CMN_MAX_DIMENSION 12 +/* Some implementations use a mesh larger than the architectural max of 12 */ +#define CMN_MAX_DIMENSION 14 #define CMN_MAX_XPS (CMN_MAX_DIMENSION * CMN_MAX_DIMENSION) #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4) @@ -47,11 +49,16 @@ #define CMN_CFGM_INFO_GLOBAL 0x0900 #define CMN_INFO_MULTIPLE_DTM_EN BIT_ULL(63) +#define CMN_S3_R2_MULTIPLE_DTM_EN BIT_ULL(59) #define CMN_INFO_RSP_VC_NUM GENMASK_ULL(53, 52) #define CMN_INFO_DAT_VC_NUM GENMASK_ULL(51, 50) #define CMN_INFO_DEVICE_ISO_ENABLE BIT_ULL(44) #define CMN_CFGM_INFO_GLOBAL_1 0x0908 +#define CMN_S3_R2_RSP_VC_NUM GENMASK_ULL(11, 9) +#define CMN_S3_R2_DAT_VC_NUM GENMASK_ULL(8, 6) +#define CMN_S3_R2_SNP_VC_NUM GENMASK_ULL(5, 3) +#define CMN_S3_R2_REQ_VC_NUM GENMASK_ULL(2, 0) #define CMN_INFO_SNP_VC_NUM GENMASK_ULL(3, 2) #define CMN_INFO_REQ_VC_NUM GENMASK_ULL(1, 0) @@ -77,6 +84,16 @@ /* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */ #define CMN__PMU_OCCUP1_ID GENMASK_ULL(34, 32) +/* But then... */ +#define CMN__PMU_EVICT_STATE_SEL GENMASK_ULL(54, 52) +#define CMN__PMU_ENHANCED_HBT_LBT_SEL GENMASK_ULL(51, 48) +#define CMN__PMU_SNP_VC_SEL GENMASK_ULL(47, 46) +#define CMN__S3_R2_CBUSY_SNTHROTTLE_SEL GENMASK_ULL(45, 42) +#define CMN__S3_R2_SN_HOME_SEL GENMASK_ULL(41, 40) +#define CMN__S3_R2_HBT_LBT_SEL GENMASK_ULL(39, 38) +#define CMN__S3_R2_CLASS_OCCUP_ID GENMASK_ULL(37, 36) +#define CMN__S3_R2_OCCUP1_ID GENMASK_ULL(35, 32) + /* Some types are designed to coexist with another device in the same node */ #define CMN_CCLA_PMU_EVENT_SEL 0x008 #define CMN_HNP_PMU_EVENT_SEL 0x008 @@ -162,15 +179,17 @@ /* Event attributes */ #define CMN_CONFIG_TYPE GENMASK_ULL(15, 0) #define CMN_CONFIG_EVENTID GENMASK_ULL(26, 16) -#define CMN_CONFIG_OCCUPID GENMASK_ULL(30, 27) +#define CMN_CONFIG_FILTER GENMASK_ULL(30, 27) #define CMN_CONFIG_BYNODEID BIT_ULL(31) #define CMN_CONFIG_NODEID GENMASK_ULL(47, 32) +#define CMN_CONFIG_FILTER2 GENMASK_ULL(51, 48) #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config) #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config) -#define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config) +#define CMN_EVENT_FILTER(event) FIELD_GET(CMN_CONFIG_FILTER, (event)->attr.config) #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config) #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config) +#define CMN_EVENT_FILTER2(event) FIELD_GET(CMN_CONFIG_FILTER2, (event)->attr.config) #define CMN_CONFIG_WP_COMBINE GENMASK_ULL(30, 27) #define CMN_CONFIG_WP_DEV_SEL GENMASK_ULL(50, 48) @@ -199,12 +218,14 @@ enum cmn_model { CMN650 = 2, CI700 = 4, CMN700 = 8, - CMNS3 = 16, + CMNS3R01 = 16, + CMNS3R2 = 32, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, CMN_700ON = ~(CMN700 - 1), CMN_650ON = CMN_700ON | CMN650, + CMNS3 = CMNS3R01 | CMNS3R2, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -215,6 +236,8 @@ enum cmn_part { PART_CMN700 = 0x43c, PART_CI700 = 0x43a, PART_CMN_S3 = 0x43e, + /* Synthetic part number, overridden to PART_CMN_S3 during discovery */ + PART_GRAVITON5 = 0xa5, }; /* CMN-600 r0px shouldn't exist in silicon, thankfully */ @@ -238,6 +261,10 @@ enum cmn_revision { REV_CMNS3_R0P0 = 0, REV_CMNS3_R0P1, REV_CMNS3_R1P0, + REV_CMNS3_R2P0, + REV_CMNS3_R2P1, + REV_CMNS3_R2P2, + REV_CMNS3_R2P5, REV_CI700_R0P0 = 0, REV_CI700_R1P0, REV_CI700_R2P0, @@ -275,12 +302,15 @@ enum cmn_node_type { }; enum cmn_filter_select { - SEL_NONE = -1, - SEL_OCCUP1ID, + SEL_NONE, + SEL_OCCUP1_ID, SEL_CLASS_OCCUP_ID, SEL_CBUSY_SNTHROTTLE_SEL, SEL_HBT_LBT_SEL, SEL_SN_HOME_SEL, + SEL_SNP_VC_SEL, + SEL_ENHANCED_HBT_LBT_SEL, + SEL_EVICT_STATE_SEL, SEL_MAX }; @@ -298,7 +328,7 @@ struct arm_cmn_node { struct { u8 val : 4; u8 count : 4; - } occupid[SEL_MAX]; + } filter[SEL_MAX]; union { u8 event[4]; __le32 event_sel; @@ -421,7 +451,9 @@ static enum cmn_model arm_cmn_model(const struct arm_cmn *cmn) case PART_CI700: return CI700; case PART_CMN_S3: - return CMNS3; + if (cmn->rev >= REV_CMNS3_R2P0) + return CMNS3R2; + return CMNS3R01; default: return 0; }; @@ -596,19 +628,82 @@ static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {} #endif +enum cmn_filter_type { + FILT_NONE, + FILT_OCCUP1_ID, + FILT_HNF_700, + FILT_HNS, + FILT_HNS_S3R2, +}; +#define CMN_FILTER(_sel) [SEL_##_sel] = CMN__PMU_##_sel +#define CMN_FILTER_V2(_sel) [SEL_##_sel] = CMN__S3_R2_##_sel + +static const u64 arm_cmn_filters[][SEL_MAX] = { + [FILT_NONE] = {}, + /* DVM etc. */ + [FILT_OCCUP1_ID] = { + CMN_FILTER(OCCUP1_ID) + }, + /* Newer HN-F */ + [FILT_HNF_700] = { + CMN_FILTER(OCCUP1_ID), + CMN_FILTER(CLASS_OCCUP_ID), + CMN_FILTER(CBUSY_SNTHROTTLE_SEL) + }, + /* HN-S */ + [FILT_HNS] = { + CMN_FILTER(OCCUP1_ID), + CMN_FILTER(CLASS_OCCUP_ID), + CMN_FILTER(CBUSY_SNTHROTTLE_SEL), + CMN_FILTER(HBT_LBT_SEL), + CMN_FILTER(SN_HOME_SEL) + }, + /* Newer HN-S */ + [FILT_HNS_S3R2] = { + CMN_FILTER_V2(OCCUP1_ID), + CMN_FILTER_V2(CLASS_OCCUP_ID), + CMN_FILTER_V2(CBUSY_SNTHROTTLE_SEL), + CMN_FILTER_V2(HBT_LBT_SEL), + CMN_FILTER_V2(SN_HOME_SEL), + CMN_FILTER(SNP_VC_SEL), + CMN_FILTER(ENHANCED_HBT_LBT_SEL), + CMN_FILTER(EVICT_STATE_SEL) + } +}; + +static enum cmn_filter_type arm_cmn_filter(enum cmn_node_type node, + enum cmn_model model) +{ + switch (node) { + default: + return FILT_NONE; + case CMN_TYPE_DVM: + case CMN_TYPE_CXRA: + case CMN_TYPE_CXHA: + case CMN_TYPE_CCRA: + case CMN_TYPE_CCHA: + return FILT_OCCUP1_ID; + case CMN_TYPE_HNF: + if (model < CMN700) + return FILT_OCCUP1_ID; + return FILT_HNF_700; + case CMN_TYPE_HNS: + if (model < CMNS3R2) + return FILT_HNS; + return FILT_HNS_S3R2; + }; +} + struct arm_cmn_hw_event { struct arm_cmn_node *dn; - u64 dtm_idx[DIV_ROUND_UP(CMN_MAX_NODES_PER_EVENT * 2, 64)]; + union { + unsigned long *dtm_idx; + int cc_idx; + }; + unsigned long *wp_idx; s8 dtc_idx[CMN_MAX_DTCS]; u8 num_dns; u8 dtm_offset; - - /* - * WP config registers are divided to UP and DOWN events. We need to - * keep to track only one of them. - */ - DECLARE_BITMAP(wp_idx, CMN_MAX_XPS); - bool wide_sel; enum cmn_filter_select filter_sel; }; @@ -626,34 +721,57 @@ static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event) return (struct arm_cmn_hw_event *)&event->hw; } -static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val) +#define BPL2 (BITS_PER_LONG / 2) + +static void arm_cmn_set_dtm_idx(struct arm_cmn_hw_event *hw, unsigned int pos, unsigned int val) { - x[pos / 32] |= (u64)val << ((pos % 32) * 2); + hw->dtm_idx[pos / BPL2] |= (unsigned long)val << ((pos % BPL2) * 2); } -static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos) +static unsigned int arm_cmn_get_dtm_idx(struct arm_cmn_hw_event *hw, unsigned int pos) { - return (x[pos / 32] >> ((pos % 32) * 2)) & 3; + return (hw->dtm_idx[pos / BPL2] >> ((pos % BPL2) * 2)) & 3; } -static void arm_cmn_set_wp_idx(unsigned long *wp_idx, unsigned int pos, bool val) +static unsigned long *arm_cmn_alloc_dtm_idx(void) +{ + return bitmap_zalloc(CMN_MAX_NODES_PER_EVENT * 2, GFP_KERNEL); +} + +static void arm_cmn_set_wp_idx(struct arm_cmn_hw_event *hw, unsigned int pos, bool val) { if (val) - set_bit(pos, wp_idx); + set_bit(pos, hw->wp_idx); } -static unsigned int arm_cmn_get_wp_idx(unsigned long *wp_idx, unsigned int pos) +static unsigned int arm_cmn_get_wp_idx(struct arm_cmn_hw_event *hw, unsigned int pos) { - return test_bit(pos, wp_idx); + return test_bit(pos, hw->wp_idx); } +static unsigned long *arm_cmn_alloc_wp_idx(void) +{ + return bitmap_zalloc(CMN_MAX_XPS, GFP_KERNEL); +} + +static void arm_cmn_clear_idx(struct arm_cmn_hw_event *hw) +{ + bitmap_zero(hw->dtm_idx, CMN_MAX_NODES_PER_EVENT * 2); + if (hw->wp_idx) + bitmap_zero(hw->wp_idx, CMN_MAX_XPS); +} + +struct arm_cmn_filter_attr { + enum cmn_filter_select sel; + u8 val; +}; + struct arm_cmn_event_attr { struct device_attribute attr; enum cmn_model model; enum cmn_node_type type; - enum cmn_filter_select fsel; u16 eventid; - u8 occupid; + struct arm_cmn_filter_attr filter[2]; }; struct arm_cmn_format_attr { @@ -662,24 +780,25 @@ struct arm_cmn_format_attr { int config; }; -#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid, _fsel)\ +#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _fa, _fb, _fc, _fd, ...) \ (&((struct arm_cmn_event_attr[]) {{ \ .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ .model = _model, \ .type = _type, \ .eventid = _eventid, \ - .occupid = _occupid, \ - .fsel = _fsel, \ + .filter = {{_fa, _fb}, {_fc, _fd}}, \ }})[0].attr.attr) -#define CMN_EVENT_ATTR(_model, _name, _type, _eventid) \ - _CMN_EVENT_ATTR(_model, _name, _type, _eventid, 0, SEL_NONE) +#define CMN_EVENT_ATTR(_model, _name, _type, _eventid, _filter...) \ + _CMN_EVENT_ATTR(_model, _name, _type, _eventid, ##_filter, 0, 0, 0, 0) static ssize_t arm_cmn_event_show(struct device *dev, struct device_attribute *attr, char *buf) { struct arm_cmn_event_attr *eattr; + struct arm_cmn_filter_attr *filter; eattr = container_of(attr, typeof(*eattr), attr); + filter = eattr->filter; if (eattr->type == CMN_TYPE_DTC) return sysfs_emit(buf, "type=0x%x\n", eattr->type); @@ -689,9 +808,13 @@ static ssize_t arm_cmn_event_show(struct device *dev, "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", eattr->type, eattr->eventid); - if (eattr->fsel > SEL_NONE) - return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n", - eattr->type, eattr->eventid, eattr->occupid); + if (filter[1].sel) + return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x,filter2=0x%x\n", + eattr->type, eattr->eventid, filter[0].val, filter[1].val); + + if (filter[0].sel) + return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x\n", + eattr->type, eattr->eventid, filter[0].val); return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, eattr->eventid); @@ -779,8 +902,8 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, return attr->mode; } -#define _CMN_EVENT_DVM(_model, _name, _event, _occup, _fsel) \ - _CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup, _fsel) +#define CMN_EVENT_DVM(_model, _name, _event, _filter...) \ + CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, ##_filter) #define CMN_EVENT_DTC(_name) \ CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0) #define CMN_EVENT_HNF(_model, _name, _event) \ @@ -807,35 +930,38 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, CMN_EVENT_ATTR(_model, ccha_##_name, CMN_TYPE_CCHA, _event) #define CMN_EVENT_CCLA(_name, _event) \ CMN_EVENT_ATTR(CMN_ANY, ccla_##_name, CMN_TYPE_CCLA, _event) -#define CMN_EVENT_HNS(_name, _event) \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) +#define _CMN_EVENT_HNS(_model, _name, _event, _filter...) \ + CMN_EVENT_ATTR(_model, hns_##_name, CMN_TYPE_HNS, _event, ##_filter) -#define CMN_EVENT_DVM(_model, _name, _event) \ - _CMN_EVENT_DVM(_model, _name, _event, 0, SEL_NONE) #define CMN_EVENT_DVM_OCC(_model, _name, _event) \ - _CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1ID) + CMN_EVENT_DVM(_model, _name##_all, _event, SEL_OCCUP1_ID, 0), \ + CMN_EVENT_DVM(_model, _name##_dvmop, _event, SEL_OCCUP1_ID, 1), \ + CMN_EVENT_DVM(_model, _name##_dvmsync, _event, SEL_OCCUP1_ID, 2) #define CMN_EVENT_HN_OCC(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1ID) + CMN_EVENT_ATTR(_model, _name##_all, _type, _event, SEL_OCCUP1_ID, 0), \ + CMN_EVENT_ATTR(_model, _name##_read, _type, _event, SEL_OCCUP1_ID, 1), \ + CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_OCCUP1_ID, 2), \ + CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, SEL_OCCUP1_ID, 3), \ + CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, SEL_OCCUP1_ID, 4) #define CMN_EVENT_HN_CLS(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, 0, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, 1, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, 2, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, 3, SEL_CLASS_OCCUP_ID) + CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, SEL_CLASS_OCCUP_ID, 0), \ + CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, SEL_CLASS_OCCUP_ID, 1), \ + CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, SEL_CLASS_OCCUP_ID, 2), \ + CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, SEL_CLASS_OCCUP_ID, 3) #define CMN_EVENT_HN_SNT(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, 1, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, 2, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, 3, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, 4, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 5, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 6, SEL_CBUSY_SNTHROTTLE_SEL) + CMN_EVENT_ATTR(_model, _name##_all, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 0), \ + CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 1), \ + CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 2), \ + CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 3), \ + CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 4), \ + CMN_EVENT_ATTR(_model, _name##_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 5), \ + CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 6), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_ccg_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 9), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_ccg_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 10), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 11), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 12), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 13) #define CMN_EVENT_HNF_OCC(_model, _name, _event) \ CMN_EVENT_HN_OCC(_model, hnf_##_name, CMN_TYPE_HNF, _event) @@ -844,23 +970,75 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_HNF_SNT(_model, _name, _event) \ CMN_EVENT_HN_SNT(_model, hnf_##_name, CMN_TYPE_HNF, _event) -#define CMN_EVENT_HNS_OCC(_name, _event) \ - CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1ID) +#define CMN_EVENT_HNS(_name, _event) \ + _CMN_EVENT_HNS(CMN_ANY, _name, _event) +#define CMN_EVENT_HNSR0(_name, _event) \ + _CMN_EVENT_HNS(CMN700 | CMNS3R01, _name, _event) +#define _CMN_EVENT_HNS_HBT(_model, _name, _event, _sel) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, _sel, 0), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, _sel, 1), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, _sel, 2) +#define _CMN_EVENT_HNS_HBT2(_model, _name, _event, _fsel1, f1) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 0), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 1), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 2) + +#define CMN_EVENT_HNS_OCC(_model, _name, _event) \ + CMN_EVENT_HN_OCC(_model, hns_##_name, CMN_TYPE_HNS, _event), \ + _CMN_EVENT_HNS(_model, _name##_rxsnp, _event, SEL_OCCUP1_ID, 5), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, SEL_OCCUP1_ID, 6), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, SEL_OCCUP1_ID, 7), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rnf, _event, SEL_OCCUP1_ID, 8), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rni, _event, SEL_OCCUP1_ID, 9), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_OCCUP1_ID, 10), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_OCCUP1_ID, 11) #define CMN_EVENT_HNS_CLS( _name, _event) \ CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) -#define CMN_EVENT_HNS_SNT(_name, _event) \ - CMN_EVENT_HN_SNT(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) -#define CMN_EVENT_HNS_HBT(_name, _event) \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_HBT_LBT_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 1, SEL_HBT_LBT_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 2, SEL_HBT_LBT_SEL) -#define CMN_EVENT_HNS_SNH(_name, _event) \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_SN_HOME_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, 1, SEL_SN_HOME_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, 2, SEL_SN_HOME_SEL) +#define CMN_EVENT_HNSR0_CLS( _name, _event) \ + CMN_EVENT_HN_CLS(CMN700 | CMNS3R01, hns_##_name, CMN_TYPE_HNS, _event) +#define CMN_EVENT_HNS_SNT(_model, _name, _event) \ + CMN_EVENT_HN_SNT(_model, hns_##_name, CMN_TYPE_HNS, _event) +#define CMN_EVENT_HNS_SNH(_model, _name, _event) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, SEL_SN_HOME_SEL, 0), \ + _CMN_EVENT_HNS(_model, _name##_sn, _event, SEL_SN_HOME_SEL, 1), \ + _CMN_EVENT_HNS(_model, _name##_home, _event, SEL_SN_HOME_SEL, 2) +#define CMN_EVENT_HNS_VC(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc0, _event, SEL_SNP_VC_SEL, 0), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc1, _event, SEL_SNP_VC_SEL, 1), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc2, _event, SEL_SNP_VC_SEL, 2) +#define CMN_EVENT_HNS_ENHBT(_name, _event) \ + _CMN_EVENT_HNS_HBT(CMNS3R2, _name, _event, SEL_ENHANCED_HBT_LBT_SEL), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rnf, _event, SEL_ENHANCED_HBT_LBT_SEL, 3), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rni, _event, SEL_ENHANCED_HBT_LBT_SEL, 4), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_ENHANCED_HBT_LBT_SEL, 5), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_ENHANCED_HBT_LBT_SEL, 6) +#define CMN_EVENT_HNS_EVICT(_model, _name, _event) \ + _CMN_EVENT_HNS_HBT2(_model, _name##_all, _event, SEL_EVICT_STATE_SEL, 0), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_eu, _event, SEL_EVICT_STATE_SEL, 1), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_en, _event, SEL_EVICT_STATE_SEL, 2), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_su, _event, SEL_EVICT_STATE_SEL, 3), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_sn, _event, SEL_EVICT_STATE_SEL, 4), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_mu, _event, SEL_EVICT_STATE_SEL, 5), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_mn, _event, SEL_EVICT_STATE_SEL, 6) + +#define CMN_EVENT_HNSR0_HBT(_name, _event) \ + _CMN_EVENT_HNS_HBT(CMN700 | CMNS3R01, _name, _event, SEL_HBT_LBT_SEL) +#define CMN_EVENT_HNS_R2SNH(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + CMN_EVENT_HNS_SNH(CMNS3R2, _name, _event) +#define CMN_EVENT_HNS_R2HBT(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + _CMN_EVENT_HNS_HBT(CMNS3R2, _name, _event, SEL_HBT_LBT_SEL) +#define CMN_EVENT_HNS_HBT_ENHBT(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_ENHBT(_name, _event) +#define CMN_EVENT_HNS_HBT_OCC(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_OCC(CMNS3R2, _name, _event) +#define CMN_EVENT_HNS_HBT_EVICT(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_EVICT(CMNS3R2, _name, _event) #define _CMN_EVENT_XP_MESH(_name, _event) \ __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \ @@ -1210,65 +1388,72 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_CCLA(pfwd_sndr_stalls_static_crd, 0x2a), CMN_EVENT_CCLA(pfwd_sndr_stalls_dynmaic_crd, 0x2b), - CMN_EVENT_HNS_HBT(cache_miss, 0x01), - CMN_EVENT_HNS_HBT(slc_sf_cache_access, 0x02), - CMN_EVENT_HNS_HBT(cache_fill, 0x03), - CMN_EVENT_HNS_HBT(pocq_retry, 0x04), - CMN_EVENT_HNS_HBT(pocq_reqs_recvd, 0x05), - CMN_EVENT_HNS_HBT(sf_hit, 0x06), - CMN_EVENT_HNS_HBT(sf_evictions, 0x07), - CMN_EVENT_HNS(dir_snoops_sent, 0x08), - CMN_EVENT_HNS(brd_snoops_sent, 0x09), - CMN_EVENT_HNS_HBT(slc_eviction, 0x0a), - CMN_EVENT_HNS_HBT(slc_fill_invalid_way, 0x0b), - CMN_EVENT_HNS(mc_retries_local, 0x0c), - CMN_EVENT_HNS_SNH(mc_reqs_local, 0x0d), + CMN_EVENT_HNS_HBT_ENHBT(cache_miss, 0x01), + CMN_EVENT_HNS_HBT_ENHBT(slc_sf_cache_access, 0x02), + CMN_EVENT_HNS_HBT_ENHBT(cache_fill, 0x03), + CMN_EVENT_HNS_HBT_OCC(pocq_retry, 0x04), + CMN_EVENT_HNS_HBT_OCC(pocq_reqs_recvd, 0x05), + CMN_EVENT_HNS_HBT_ENHBT(sf_hit, 0x06), + CMN_EVENT_HNS_HBT_EVICT(sf_evictions, 0x07), + CMN_EVENT_HNS_VC(dir_snoops_sent, 0x08), + CMN_EVENT_HNS_VC(brd_snoops_sent, 0x09), + CMN_EVENT_HNS_HBT_EVICT(slc_eviction, 0x0a), + CMN_EVENT_HNS_HBT_ENHBT(slc_fill_invalid_way, 0x0b), + CMN_EVENT_HNS_R2SNH(mc_retries_local, 0x0c), + CMN_EVENT_HNS_SNH(CMN_ANY, mc_reqs_local, 0x0d), CMN_EVENT_HNS(qos_hh_retry, 0x0e), - CMN_EVENT_HNS_OCC(qos_pocq_occupancy, 0x0f), - CMN_EVENT_HNS(pocq_addrhaz, 0x10), - CMN_EVENT_HNS(pocq_atomic_addrhaz, 0x11), - CMN_EVENT_HNS(ld_st_swp_adq_full, 0x12), - CMN_EVENT_HNS(cmp_adq_full, 0x13), + CMN_EVENT_HNS_OCC(CMN_ANY, qos_pocq_occupancy, 0x0f), + CMN_EVENT_HNS_HBT_ENHBT(pocq_addrhaz, 0x10), + CMN_EVENT_HNS_HBT_ENHBT(pocq_atomic_addrhaz, 0x11), + CMN_EVENT_HNSR0(ld_st_swp_adq_full, 0x12), + CMN_EVENT_HNSR0(cmp_adq_full, 0x13), CMN_EVENT_HNS(txdat_stall, 0x14), CMN_EVENT_HNS(txrsp_stall, 0x15), - CMN_EVENT_HNS(seq_full, 0x16), + CMN_EVENT_HNSR0(seq_full, 0x16), CMN_EVENT_HNS(seq_hit, 0x17), - CMN_EVENT_HNS(snp_sent, 0x18), - CMN_EVENT_HNS(sfbi_dir_snp_sent, 0x19), - CMN_EVENT_HNS(sfbi_brd_snp_sent, 0x1a), + CMN_EVENT_HNS_VC(snp_sent, 0x18), + CMN_EVENT_HNS_VC(sfbi_dir_snp_sent, 0x19), + CMN_EVENT_HNS_VC(sfbi_brd_snp_sent, 0x1a), CMN_EVENT_HNS(intv_dirty, 0x1c), - CMN_EVENT_HNS(stash_snp_sent, 0x1d), - CMN_EVENT_HNS(stash_data_pull, 0x1e), - CMN_EVENT_HNS(snp_fwded, 0x1f), - CMN_EVENT_HNS(atomic_fwd, 0x20), + CMN_EVENT_HNSR0(stash_snp_sent, 0x1d), + CMN_EVENT_HNSR0(stash_data_pull, 0x1e), + CMN_EVENT_HNS_VC(snp_fwded, 0x1f), + CMN_EVENT_HNSR0(atomic_fwd, 0x20), CMN_EVENT_HNS(mpam_hardlim, 0x21), CMN_EVENT_HNS(mpam_softlim, 0x22), - CMN_EVENT_HNS(snp_sent_cluster, 0x23), - CMN_EVENT_HNS(sf_imprecise_evict, 0x24), + CMN_EVENT_HNS_VC(snp_sent_cluster, 0x23), + CMN_EVENT_HNS_R2HBT(sf_imprecise_evict, 0x24), CMN_EVENT_HNS(sf_evict_shared_line, 0x25), CMN_EVENT_HNS_CLS(pocq_class_occup, 0x26), CMN_EVENT_HNS_CLS(pocq_class_retry, 0x27), CMN_EVENT_HNS_CLS(class_mc_reqs_local, 0x28), - CMN_EVENT_HNS_CLS(class_cgnt_cmin, 0x29), - CMN_EVENT_HNS_SNT(sn_throttle, 0x2a), - CMN_EVENT_HNS_SNT(sn_throttle_min, 0x2b), + CMN_EVENT_HNSR0_CLS(class_cgnt_cmin, 0x29), + CMN_EVENT_HNS_SNT(CMN_ANY, sn_throttle, 0x2a), + CMN_EVENT_HNS_SNT(CMN_ANY, sn_throttle_min, 0x2b), CMN_EVENT_HNS(sf_precise_to_imprecise, 0x2c), CMN_EVENT_HNS(snp_intv_cln, 0x2d), CMN_EVENT_HNS(nc_excl, 0x2e), - CMN_EVENT_HNS(excl_mon_ovfl, 0x2f), + CMN_EVENT_HNSR0(excl_mon_ovfl, 0x2f), CMN_EVENT_HNS(snp_req_recvd, 0x30), CMN_EVENT_HNS(snp_req_byp_pocq, 0x31), CMN_EVENT_HNS(dir_ccgha_snp_sent, 0x32), CMN_EVENT_HNS(brd_ccgha_snp_sent, 0x33), - CMN_EVENT_HNS(ccgha_snp_stall, 0x34), + CMN_EVENT_HNSR0(ccgha_snp_stall, 0x34), CMN_EVENT_HNS(lbt_req_hardlim, 0x35), CMN_EVENT_HNS(hbt_req_hardlim, 0x36), CMN_EVENT_HNS(sf_reupdate, 0x37), - CMN_EVENT_HNS(excl_sf_imprecise, 0x38), + CMN_EVENT_HNS_R2HBT(excl_sf_imprecise, 0x38), CMN_EVENT_HNS(snp_pocq_addrhaz, 0x39), - CMN_EVENT_HNS(mc_retries_remote, 0x3a), - CMN_EVENT_HNS_SNH(mc_reqs_remote, 0x3b), + CMN_EVENT_HNS_R2SNH(mc_retries_remote, 0x3a), + CMN_EVENT_HNS_SNH(CMN_ANY, mc_reqs_remote, 0x3b), CMN_EVENT_HNS_CLS(class_mc_reqs_remote, 0x3c), + CMN_EVENT_HNS_ENHBT(readonce_hazard_detected, 0x3d), + CMN_EVENT_HNS_ENHBT(readonce_fwd_data_completed, 0x3e), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy00, 0x40), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy01, 0x41), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy10, 0x42), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy11, 0x43), + CMN_EVENT_HNS_ENHBT(ro_rnsd_new_alloc_hint, 0x44), NULL }; @@ -1301,9 +1486,10 @@ static ssize_t arm_cmn_format_show(struct device *dev, static struct attribute *arm_cmn_format_attrs[] = { CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE), CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID), - CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID), + CMN_FORMAT_ATTR(filter, CMN_CONFIG_FILTER), CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID), CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID), + CMN_FORMAT_ATTR(filter2, CMN_CONFIG_FILTER2), CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL), CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL), @@ -1314,6 +1500,9 @@ static struct attribute *arm_cmn_format_attrs[] = { _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL), _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK), + /* Old name for UAPI compatibility */ + CMN_FORMAT_ATTR(occupid, CMN_CONFIG_FILTER), + NULL }; @@ -1377,7 +1566,7 @@ static int arm_cmn_get_assigned_wp_idx(struct perf_event *event, struct arm_cmn_hw_event *hw, unsigned int pos) { - return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos); + return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw, pos); } static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm, @@ -1388,7 +1577,7 @@ static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm, struct arm_cmn_hw_event *hw = to_cmn_hw(event); dtm->wp_event[wp_idx] = hw->dtc_idx[dtc]; - arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event)); + arm_cmn_set_wp_idx(hw, pos, wp_idx - CMN_EVENT_EVENTID(event)); } static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx) @@ -1459,7 +1648,7 @@ static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw, dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; reg = readq_relaxed(dtm->base + offset); } - dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + dtm_idx = arm_cmn_get_dtm_idx(hw, i); count += (u16)(reg >> (dtm_idx * 16)); } return count; @@ -1506,7 +1695,7 @@ static void arm_cmn_event_read(struct perf_event *event) unsigned long flags; if (CMN_EVENT_TYPE(event) == CMN_TYPE_DTC) { - delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]); + delta = arm_cmn_read_cc(cmn->dtc + hw->cc_idx); local64_add(delta, &event->count); return; } @@ -1525,30 +1714,23 @@ static void arm_cmn_event_read(struct perf_event *event) } static int arm_cmn_set_event_sel_hi(struct arm_cmn_node *dn, - enum cmn_filter_select fsel, u8 occupid) + enum cmn_filter_select fsel, u8 val) { - u64 reg; + if (!dn->filter[fsel].count) { + const u64 *filter = arm_cmn_filters[dn->filter[SEL_NONE].val]; + u64 reg = 0; - if (fsel == SEL_NONE) - return 0; + dn->filter[fsel].val = val; + for (int i = SEL_OCCUP1_ID; i < SEL_MAX; i++) { + if (filter[i]) + reg |= field_prep(filter[i], dn->filter[i].val); + } - if (!dn->occupid[fsel].count) { - dn->occupid[fsel].val = occupid; - reg = FIELD_PREP(CMN__PMU_CBUSY_SNTHROTTLE_SEL, - dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) | - FIELD_PREP(CMN__PMU_SN_HOME_SEL, - dn->occupid[SEL_SN_HOME_SEL].val) | - FIELD_PREP(CMN__PMU_HBT_LBT_SEL, - dn->occupid[SEL_HBT_LBT_SEL].val) | - FIELD_PREP(CMN__PMU_CLASS_OCCUP_ID, - dn->occupid[SEL_CLASS_OCCUP_ID].val) | - FIELD_PREP(CMN__PMU_OCCUP1_ID, - dn->occupid[SEL_OCCUP1ID].val); writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); - } else if (dn->occupid[fsel].val != occupid) { + } else if (dn->filter[fsel].val != val) { return -EBUSY; } - dn->occupid[fsel].count++; + dn->filter[fsel].count++; return 0; } @@ -1573,7 +1755,7 @@ static void arm_cmn_event_start(struct perf_event *event, int flags) int i; if (type == CMN_TYPE_DTC) { - struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; + struct arm_cmn_dtc *dtc = cmn->dtc + hw->cc_idx; writel_relaxed(CMN_DT_DTC_CTL_DT_EN | CMN_DT_DTC_CTL_CG_DISABLE, dtc->base + CMN_DT_DTC_CTL); @@ -1591,7 +1773,7 @@ static void arm_cmn_event_start(struct perf_event *event, int flags) writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx)); } } else for_each_hw_dn(hw, dn, i) { - int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + int dtm_idx = arm_cmn_get_dtm_idx(hw, i); arm_cmn_set_event_sel_lo(dn, dtm_idx, CMN_EVENT_EVENTID(event), hw->wide_sel); @@ -1607,7 +1789,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) int i; if (type == CMN_TYPE_DTC) { - struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; + struct arm_cmn_dtc *dtc = cmn->dtc + hw->cc_idx; dtc->cc_active = false; writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); @@ -1620,7 +1802,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx)); } } else for_each_hw_dn(hw, dn, i) { - int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + int dtm_idx = arm_cmn_get_dtm_idx(hw, i); arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel); } @@ -1630,7 +1812,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) struct arm_cmn_val { u8 dtm_count[CMN_MAX_DTMS]; - u8 occupid[CMN_MAX_DTMS][SEL_MAX]; + u8 filter[CMN_MAX_DTMS][SEL_MAX]; u8 wp[CMN_MAX_DTMS][4]; u8 wp_combine[CMN_MAX_DTMS][2]; int dtc_count[CMN_MAX_DTCS]; @@ -1674,8 +1856,10 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, val->dtm_count[dtm]++; - if (sel > SEL_NONE) - val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1; + if (sel) + val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1; + if (sel == SEL_EVICT_STATE_SEL) + val->filter[dtm][SEL_HBT_LBT_SEL] = CMN_EVENT_FILTER2(event) + 1; if (type != CMN_TYPE_WP) continue; @@ -1726,8 +1910,12 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) goto done; - if (sel > SEL_NONE && val->occupid[dtm][sel] && - val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1) + if (sel && val->filter[dtm][sel] && + val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1) + goto done; + + if (sel == SEL_EVICT_STATE_SEL && val->filter[dtm][SEL_HBT_LBT_SEL] && + val->filter[dtm][SEL_HBT_LBT_SEL] != CMN_EVENT_FILTER2(event) + 1) goto done; if (type != CMN_TYPE_WP) @@ -1748,9 +1936,9 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) return ret; } -static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn, - enum cmn_node_type type, - unsigned int eventid) +static enum cmn_filter_select arm_cmn_event_filter(const struct arm_cmn *cmn, + enum cmn_node_type type, + unsigned int eventid) { struct arm_cmn_event_attr *e; enum cmn_model model = arm_cmn_model(cmn); @@ -1758,12 +1946,20 @@ static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn, for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr); if (e->model & model && e->type == type && e->eventid == eventid) - return e->fsel; + return e->filter[0].sel; } return SEL_NONE; } +static void arm_cmn_event_destroy(struct perf_event *event) +{ + struct arm_cmn_hw_event *hw = to_cmn_hw(event); + + bitmap_free(hw->dtm_idx); + bitmap_free(hw->wp_idx); +} + static int arm_cmn_event_init(struct perf_event *event) { struct arm_cmn *cmn = to_cmn(event->pmu); @@ -1788,6 +1984,11 @@ static int arm_cmn_event_init(struct perf_event *event) if (type == CMN_TYPE_DTC) return arm_cmn_validate_group(cmn, event); + event->destroy = arm_cmn_event_destroy; + hw->dtm_idx = arm_cmn_alloc_dtm_idx(); + if (!hw->dtm_idx) + return -ENOMEM; + eventid = CMN_EVENT_EVENTID(event); /* For watchpoints we need the actual XP node here */ if (type == CMN_TYPE_WP) { @@ -1798,6 +1999,9 @@ static int arm_cmn_event_init(struct perf_event *event) /* ...but the DTM may depend on which port we're watching */ if (cmn->multi_dtm) hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; + hw->wp_idx = arm_cmn_alloc_wp_idx(); + if (!hw->wp_idx) + return -ENOMEM; } else if (type == CMN_TYPE_XP && (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) { hw->wide_sel = true; @@ -1807,7 +2011,7 @@ static int arm_cmn_event_init(struct perf_event *event) } /* This is sufficiently annoying to recalculate, so cache it */ - hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid); + hw->filter_sel = arm_cmn_event_filter(cmn, type, eventid); bynodeid = CMN_EVENT_BYNODEID(event); nodeid = CMN_EVENT_NODEID(event); @@ -1848,7 +2052,7 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, while (i--) { struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; - unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + unsigned int dtm_idx = arm_cmn_get_dtm_idx(hw, i); if (type == CMN_TYPE_WP) { int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i); @@ -1856,19 +2060,36 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, dtm->wp_event[wp_idx] = -1; } - if (hw->filter_sel > SEL_NONE) - hw->dn[i].occupid[hw->filter_sel].count--; + if (hw->filter_sel) + hw->dn[i].filter[hw->filter_sel].count--; + if (hw->filter_sel == SEL_EVICT_STATE_SEL) + hw->dn[i].filter[SEL_HBT_LBT_SEL].count--; dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); } - memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); - memset(hw->wp_idx, 0, sizeof(hw->wp_idx)); + arm_cmn_clear_idx(hw); for_each_hw_dtc_idx(hw, j, idx) cmn->dtc[j].counters[idx] = NULL; } +static int arm_cmn_set_event_filter(struct arm_cmn_node *dn, struct perf_event *event) +{ + enum cmn_filter_select fsel = to_cmn_hw(event)->filter_sel; + int ret = 0; + + if (fsel) + ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER(event)); + + if (fsel == SEL_EVICT_STATE_SEL && !ret) { + ret = arm_cmn_set_event_sel_hi(dn, SEL_HBT_LBT_SEL, CMN_EVENT_FILTER2(event)); + if (ret) + dn->filter[fsel].count--; + } + return ret; +} + static int arm_cmn_event_add(struct perf_event *event, int flags) { struct arm_cmn *cmn = to_cmn(event->pmu); @@ -1883,7 +2104,7 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) return -ENOSPC; cmn->dtc[i].cycles = event; - hw->dtc_idx[0] = i; + hw->cc_idx = i; if (flags & PERF_EF_START) arm_cmn_event_start(event, 0); @@ -1944,11 +2165,11 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx + (nid.port << 4) + (nid.dev << 2); - if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event))) + if (arm_cmn_set_event_filter(dn, event)) goto free_dtms; } - arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); + arm_cmn_set_dtm_idx(hw, i, dtm_idx); dtm->input_sel[dtm_idx] = input_sel; shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx); @@ -1981,7 +2202,7 @@ static void arm_cmn_event_del(struct perf_event *event, int flags) arm_cmn_event_stop(event, PERF_EF_UPDATE); if (type == CMN_TYPE_DTC) - cmn->dtc[hw->dtc_idx[0]].cycles = NULL; + cmn->dtc[hw->cc_idx].cycles = NULL; else arm_cmn_event_clear(cmn, event, hw->num_dns); } @@ -2222,6 +2443,18 @@ static unsigned int arm_cmn_dtc_domain(struct arm_cmn *cmn, void __iomem *xp_reg return FIELD_GET(CMN_DTM_UNIT_INFO_DTC_DOMAIN, readl_relaxed(xp_region + offset)); } +static unsigned int arm_cmn_graviton5_dtc_domain(u16 xp_id) +{ + unsigned int x = (xp_id >> 7) & 0xf; + unsigned int y = (xp_id >> 3) & 0xf; + + /* + * The unit info register reads as zero; derive the DTC domain from + * the XP's mesh coordinates over the 10x14 mesh. + */ + return (x / 5) + (y / 7) * 2; +} + static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node) { int level; @@ -2262,11 +2495,13 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) void __iomem *cfg_region, __iomem *xp_region; struct arm_cmn_node cfg, *dn; struct arm_cmn_dtm *dtm; + enum cmn_model model; enum cmn_part part; u16 child_count, child_poff; u64 reg; int i, j; size_t sz; + bool graviton5_workaround = false; arm_cmn_init_node_info(cmn, rgn_offset, &cfg); if (cfg.type != CMN_TYPE_CFG) @@ -2277,6 +2512,13 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01); part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg); part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8; + + /* Graviton5 has a customised CMN-S3 which needs some fixups */ + if (cmn->part == PART_GRAVITON5) { + cmn->part = PART_CMN_S3; + graviton5_workaround = true; + } + /* 600AE is close enough that it's not really worth more complexity */ if (part == PART_CMN600AE) part = PART_CMN600; @@ -2285,30 +2527,44 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n", cmn->part, part); cmn->part = part; - if (!arm_cmn_model(cmn)) - dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_23); cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); + model = arm_cmn_model(cmn); + if (!model) + dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); + /* * With the device isolation feature, if firmware has neglected to enable * an XP port then we risk locking up if we try to access anything behind - * it; however we also have no way to tell from Non-Secure whether any - * given port is disabled or not, so the only way to win is not to play... + * it; however prior to CMN S3 r2p0 we also have no way to tell from + * Non-Secure whether any given port is disabled or not, so in that case + * the only way to win is not to play... */ reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL); - if (reg & CMN_INFO_DEVICE_ISO_ENABLE) { + if (reg & CMN_INFO_DEVICE_ISO_ENABLE && model == CMNS3R01) { dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n"); return -ENODEV; } - cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; - cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); - cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); + if (model < CMNS3R2) { + cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; + cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); + cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); + } else { + cmn->multi_dtm = reg & CMN_S3_R2_MULTIPLE_DTM_EN; + } reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL_1); - cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); - cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); + if (model < CMNS3R2) { + cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); + cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); + } else { + cmn->rsp_vc_num = FIELD_GET(CMN_S3_R2_RSP_VC_NUM, reg); + cmn->dat_vc_num = FIELD_GET(CMN_S3_R2_DAT_VC_NUM, reg); + cmn->snp_vc_num = FIELD_GET(CMN_S3_R2_SNP_VC_NUM, reg); + cmn->req_vc_num = FIELD_GET(CMN_S3_R2_REQ_VC_NUM, reg); + } reg = readq_relaxed(cfg_region + CMN_CHILD_INFO); child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); @@ -2366,6 +2622,8 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) if (cmn->part == PART_CMN600) xp->dtc = -1; + else if (graviton5_workaround) + xp->dtc = arm_cmn_graviton5_dtc_domain(xp->id); else xp->dtc = arm_cmn_dtc_domain(cmn, xp_region); @@ -2406,15 +2664,12 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(xp_region + child_poff + j * 8); /* * Don't even try to touch anything external, since in general - * we haven't a clue how to power up arbitrary CHI requesters. - * As of CMN-600r1 these could only be RN-SAMs or CXLAs, - * neither of which have any PMU events anyway. - * (Actually, CXLAs do seem to have grown some events in r1p2, - * but they don't go to regular XP DTMs, and they depend on - * secure configuration which we can't easily deal with) + * we haven't a clue how to power up arbitrary CHI requesters, + * and none of them have standard PMU events anyway. Isolated + * nodes effectively just do not exist at all from our PoV. */ - if (reg & CMN_CHILD_NODE_EXTERNAL) { - dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); + if (reg & (CMN_CHILD_NODE_EXTERNAL | CMN_CHILD_NODE_ISOLATED)) { + dev_dbg(cmn->dev, "ignoring external/isolated node %llx\n", reg); continue; } /* @@ -2441,9 +2696,18 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid); return -ENODEV; } + /* + * We can utilise the "wasted" filter array slot to store + * the index for referencing the filter encodings later. + */ + dn->filter[SEL_NONE].val = arm_cmn_filter(dn->type, model); switch (dn->type) { case CMN_TYPE_DTC: + if (graviton5_workaround) { + /* Node info logical ID is zeroed; use the XP's */ + dn->logid = xp->logid; + } cmn->num_dtcs++; dn++; break; @@ -2659,6 +2923,7 @@ static const struct acpi_device_id arm_cmn_acpi_match[] = { { "ARMHC650" }, { "ARMHC700" }, { "ARMHC003" }, + { "AMZN0070", PART_GRAVITON5 }, {} }; MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match); diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index 80fb314d5135..60005f89892d 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -78,13 +78,13 @@ static struct acpi_apmt_node *arm_cspmu_apmt_node(struct device *dev) } /* - * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except - * counter register. The counter register can be implemented as 32-bit or 64-bit - * register depending on the value of PMCFGR.SIZE field. For 64-bit access, - * single-copy 64-bit atomic support is implementation defined. APMT node flag - * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit - * single copy atomic is not supported, the driver treats the register as a pair - * of 32-bit register. + * With FEAT_CSPMU_EXT32, all of the MMIO registers are 32-bit except the + * counter registers, which are either 32-bit or 64-bit depending on the value + * of PMCFGR.SIZE. It is implementation-defined whether single-copy-atomic + * 64-bit accesses are supported, so we rely on a firmware flag to identify + * that, and otherwise treat a 64-bit counter as a non-atomic pair of 32-bit + * registers. With FEAT_CSPMU_EXT64, everything is 64-bit, but we may still + * have to deal with atomicity being broken. */ /* @@ -173,13 +173,30 @@ arm_cspmu_event_attr_is_visible(struct kobject *kobj, eattr = container_of(attr, typeof(*eattr), attr.attr); /* Hide cycle event if not supported */ - if (!supports_cycle_counter(cspmu) && + if ((cspmu->has_ext64 || !supports_cycle_counter(cspmu)) && eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT) return 0; return attr->mode; } +ssize_t arm_cspmu_default_format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct perf_pmu_events_attr *fmt = container_of(attr, typeof(*fmt), attr); + struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev)); + u64 field = cspmu->has_ext64 ? U64_MAX : U32_MAX; + DECLARE_BITMAP(bits, 64) = { BITMAP_FROM_U64(field) }; + + if (!fmt->id) { + set_bit(32, bits); /* For 32-bit "cycles" event */ + return sysfs_emit(buf, "config:%*pbl\n", 64, bits); + } + + return sysfs_emit(buf, "config%lld:%*pbl\n", fmt->id, 64, bits); +} +EXPORT_SYMBOL_GPL(arm_cspmu_default_format_show); + static struct attribute *arm_cspmu_format_attrs[] = { ARM_CSPMU_FORMAT_EVENT_ATTR, ARM_CSPMU_FORMAT_FILTER_ATTR, @@ -198,9 +215,9 @@ arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) return attrs; } -static u32 arm_cspmu_event_type(const struct perf_event *event) +static u64 arm_cspmu_event_type(const struct perf_event *event) { - return event->attr.config & ARM_CSPMU_EVENT_MASK; + return event->attr.config; } static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event) @@ -208,6 +225,16 @@ static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event) return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT); } +static u64 arm_cspmu_filter(const struct perf_event *event) +{ + return event->attr.config1; +} + +static u64 arm_cspmu_filter2(const struct perf_event *event) +{ + return event->attr.config2; +} + static ssize_t arm_cspmu_identifier_show(struct device *dev, struct device_attribute *attr, char *page) @@ -250,38 +277,43 @@ static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu) struct device *dev; struct acpi_apmt_node *apmt_node; u8 pmu_type; - char *name; char acpi_hid_string[ACPI_ID_LEN] = { 0 }; - static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 }; + static atomic_t pmu_idx; + u32 id; dev = cspmu->dev; apmt_node = arm_cspmu_apmt_node(dev); if (!apmt_node) return devm_kasprintf(dev, GFP_KERNEL, PMUNAME "_%u", - atomic_fetch_inc(&pmu_idx[0])); + atomic_fetch_inc(&pmu_idx)); pmu_type = apmt_node->type; - - if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) { + switch (pmu_type) { + default: dev_err(dev, "unsupported PMU type-%u\n", pmu_type); return NULL; - } - - if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) { + case ACPI_APMT_NODE_TYPE_ACPI: memcpy(acpi_hid_string, &apmt_node->inst_primary, sizeof(apmt_node->inst_primary)); - name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME, + return devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME, arm_cspmu_type_str[pmu_type], acpi_hid_string, apmt_node->inst_secondary); - } else { - name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME, - arm_cspmu_type_str[pmu_type], - atomic_fetch_inc(&pmu_idx[pmu_type])); + case ACPI_APMT_NODE_TYPE_MC: + id = apmt_node->id; + break; + case ACPI_APMT_NODE_TYPE_SMMU: + case ACPI_APMT_NODE_TYPE_PCIE_ROOT: + id = apmt_node->inst_primary; + break; + case ACPI_APMT_NODE_TYPE_CACHE: + id = apmt_node->inst_secondary; + break; } - return name; + return devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%u", PMUNAME, + arm_cspmu_type_str[pmu_type], id); } static ssize_t arm_cspmu_cpumask_show(struct device *dev, @@ -412,6 +444,17 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) DEFAULT_IMPL_OP(event_attr_is_visible), }; + /* + * With 64-bit events, since our default "cycles" encoding won't work, + * and the architecture recommends against implementing it anyway, we + * choose to effectively ignore FEAT_CSPMU_CCNTR, unless a vendor + * module really wants to provide its own encoding and ops. + */ + if (cspmu->has_ext64) { + cspmu->impl.ops.is_cycle_counter_event = NULL; + cspmu->impl.ops.set_cc_filter = NULL; + } + /* Firmware may override implementer/product ID from PMIIDR */ if (apmt_node && apmt_node->impl_id) cspmu->impl.pmiidr = apmt_node->impl_id; @@ -432,13 +475,15 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) if (ret) module_put(match->module); } else { - WARN(1, "arm_cspmu failed to get module: %s\n", + dev_WARN(cspmu->dev, "Failed to get module: %s\n", match->module_name); ret = -EINVAL; } } else { request_module_nowait(match->module_name); - ret = -EPROBE_DEFER; + ret = dev_err_probe(cspmu->dev, -EPROBE_DEFER, + "Waiting for module %s to load\n", + match->module_name); } mutex_unlock(&arm_cspmu_lock); @@ -511,19 +556,24 @@ static int arm_cspmu_alloc_attr_groups(struct arm_cspmu *cspmu) return 0; } +static inline int arm_cspmu_pmcr(struct arm_cspmu *cspmu) +{ + return cspmu->has_ext64 ? PMCR_64 : PMCR; +} + static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu) { - writel(PMCR_C | PMCR_P, cspmu->base0 + PMCR); + writel(PMCR_C | PMCR_P, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu) { - writel(PMCR_E, cspmu->base0 + PMCR); + writel(PMCR_E, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu) { - writel(0, cspmu->base0 + PMCR); + writel(0, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static void arm_cspmu_enable(struct pmu *pmu) @@ -554,7 +604,8 @@ static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events, struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu); if (supports_cycle_counter(cspmu)) { - if (cspmu->impl.ops.is_cycle_counter_event(event)) { + if (cspmu->impl.ops.is_cycle_counter_event && + cspmu->impl.ops.is_cycle_counter_event(event)) { /* Search for available cycle counter. */ if (test_and_set_bit(cspmu->cycle_counter_logical_idx, hw_events->used_ctrs)) @@ -797,26 +848,33 @@ static void arm_cspmu_event_update(struct perf_event *event) static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu, struct hw_perf_event *hwc) { - u32 offset = PMEVTYPER + (4 * hwc->idx); - - writel(hwc->config, cspmu->base0 + offset); + if (cspmu->has_ext64) + writeq(hwc->config, cspmu->base0 + PMEVTYPER + (8 * hwc->idx)); + else + writel(hwc->config, cspmu->base0 + PMEVTYPER + (4 * hwc->idx)); } static void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu, const struct perf_event *event) { - u32 filter = event->attr.config1 & ARM_CSPMU_FILTER_MASK; - u32 filter2 = event->attr.config2 & ARM_CSPMU_FILTER_MASK; - u32 offset = 4 * event->hw.idx; + u64 filter = arm_cspmu_filter(event); + u64 filter2 = arm_cspmu_filter2(event); + int n = event->hw.idx; - writel(filter, cspmu->base0 + PMEVFILTR + offset); - writel(filter2, cspmu->base0 + PMEVFILT2R + offset); + if (cspmu->has_ext64) { + writeq(filter, cspmu->base0 + PMEVFILTR + (8 * n)); + writeq(filter2, cspmu->base0 + PMEVFILT2R + (8 * n)); + } else { + writel(filter, cspmu->base0 + PMEVFILTR + (4 * n)); + writel(filter2, cspmu->base0 + PMEVFILT2R + (4 * n)); + } } +/* Note we deliberately don't expect 64-bit filters here; see init_impl_ops */ static void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, const struct perf_event *event) { - u32 filter = event->attr.config1 & ARM_CSPMU_FILTER_MASK; + u32 filter = arm_cspmu_filter(event); writel(filter, cspmu->base0 + PMCCFILTR); } @@ -969,6 +1027,30 @@ static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu) } } + /* + * We can infer FEAT_CSPMU_EXT64 from PMCNTEN, or hope that anything + * that failed to get that right has at least implemented the optional + * PMDEVARCH correctly... + * + * Note that architecturally, has_ext64 *should* imply has_atomic_dword, + * but enough implementations have ignored that already that we'll just + * have to still rely on the firmware flag. + */ + writel(~0U, cspmu->base0 + PMCNTENCLR); + writel(~0U, cspmu->base0 + PMCNTEN); + if (readl(cspmu->base0 + PMCNTENCLR)) { + cspmu->has_ext64 = true; + writel(0, cspmu->base0 + PMCNTEN); + } else { + u32 reg = readl(cspmu->base0 + PMDEVARCH); + + if (reg & ARM_CSPMU_PMDEVARCH_PRESENT) { + reg &= ARM_CSPMU_PMDEVARCH_ARCHPART; + if (reg == 0xaf4 || reg == 0xaf5) + cspmu->has_ext64 = true; + } + } + cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR); cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1; @@ -1070,10 +1152,8 @@ static int arm_cspmu_request_irq(struct arm_cspmu *cspmu) ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev), cspmu); - if (ret) { - dev_err(dev, "Could not request IRQ %d\n", irq); + if (ret) return ret; - } cspmu->irq = irq; @@ -1249,8 +1329,11 @@ static int arm_cspmu_device_probe(struct platform_device *pdev) return ret; ret = arm_cspmu_request_irq(cspmu); - if (ret) - return ret; + if (ret) { + if (counter_size(cspmu) < 64) + return ret; + dev_info(cspmu->dev, "Continuing without IRQ\n"); + } ret = arm_cspmu_get_cpus(cspmu); if (ret) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h index 3fc5c8d77266..c4058d602477 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.h +++ b/drivers/perf/arm_cspmu/arm_cspmu.h @@ -35,21 +35,15 @@ PMU_EVENT_ATTR_ID(_name, arm_cspmu_sysfs_event_show, _config) -/* Default event id mask */ -#define ARM_CSPMU_EVENT_MASK GENMASK_ULL(63, 0) - -/* Default filter value mask */ -#define ARM_CSPMU_FILTER_MASK GENMASK_ULL(63, 0) - /* Default event format */ #define ARM_CSPMU_FORMAT_EVENT_ATTR \ - ARM_CSPMU_FORMAT_ATTR(event, "config:0-32") + PMU_EVENT_ATTR_ID(event, arm_cspmu_default_format_show, 0) /* Default filter format */ #define ARM_CSPMU_FORMAT_FILTER_ATTR \ - ARM_CSPMU_FORMAT_ATTR(filter, "config1:0-31") + PMU_EVENT_ATTR_ID(filter, arm_cspmu_default_format_show, 1) #define ARM_CSPMU_FORMAT_FILTER2_ATTR \ - ARM_CSPMU_FORMAT_ATTR(filter2, "config2:0-31") + PMU_EVENT_ATTR_ID(filter2, arm_cspmu_default_format_show, 2) /* * This is the default event number for cycle count, if supported, since the @@ -78,6 +72,7 @@ #define PMEVFILT2R 0x800 #define PMEVFILTR 0xA00 #define PMCNTENSET 0xC00 +#define PMCNTEN 0xC10 #define PMCNTENCLR 0xC20 #define PMINTENSET 0xC40 #define PMINTENCLR 0xC60 @@ -87,6 +82,8 @@ #define PMCFGR 0xE00 #define PMCR 0xE04 #define PMIIDR 0xE08 +#define PMCR_64 0xE10 +#define PMDEVARCH 0xFBC #define PMPIDR0 0xFE0 #define PMPIDR1 0xFE4 #define PMPIDR2 0xFE8 @@ -154,6 +151,10 @@ #define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B #define ARM_CSPMU_IMPL_ID_AMPERE 0xA16 +/* PMDEVARCH */ +#define ARM_CSPMU_PMDEVARCH_PRESENT BIT(20) +#define ARM_CSPMU_PMDEVARCH_ARCHPART GENMASK(11, 0) + struct arm_cspmu; /* This tracks the events assigned to each counter in the PMU. */ @@ -183,7 +184,7 @@ struct arm_cspmu_impl_ops { /* Check if the event corresponds to cycle count event */ bool (*is_cycle_counter_event)(const struct perf_event *event); /* Decode event type/id from configs */ - u32 (*event_type)(const struct perf_event *event); + u64 (*event_type)(const struct perf_event *event); /* Set/reset event filters */ void (*set_cc_filter)(struct arm_cspmu *cspmu, const struct perf_event *event); @@ -234,6 +235,7 @@ struct arm_cspmu { int irq; bool has_atomic_dword; + bool has_ext64; u32 pmcfgr; u32 num_logical_ctrs; u32 num_set_clr_reg; @@ -250,6 +252,9 @@ ssize_t arm_cspmu_sysfs_event_show(struct device *dev, struct device_attribute *attr, char *buf); +ssize_t arm_cspmu_default_format_show(struct device *dev, + struct device_attribute *attr, char *buf); + /* Register vendor backend. */ int arm_cspmu_impl_register(const struct arm_cspmu_impl_match *impl_match); diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c index bac83e424d6d..a4c1ab886709 100644 --- a/drivers/perf/arm_cspmu/nvidia_cspmu.c +++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c @@ -762,7 +762,7 @@ static void pcie_tgt_pmu_reset_ev_filter(struct arm_cspmu *cspmu, pcie_tgt_pmu_config_addr_filter(cspmu, false, base, mask, idx); } -static u32 pcie_tgt_pmu_event_type(const struct perf_event *event) +static u64 pcie_tgt_pmu_event_type(const struct perf_event *event) { return event->attr.config & NV_PCIE_TGT_EV_TYPE_MASK; } diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index 939bcbd433aa..aa1dac0b440f 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -931,8 +931,13 @@ int armpmu_register(struct arm_pmu *pmu) /* * By this stage we know our supported CPUs on either DT/ACPI platforms, * detect the SMT implementation. + * On SMT CPUs, the PMCCNTR_EL0 increments from the processor clock rather + * than the PE clock (ARM DDI0487 L.b D13.1.3) which means it'll continue + * counting on a WFI PE if one of its SMT sibling is not idle on a + * multi-threaded implementation. So don't use it on SMT cores. */ - pmu->has_smt = topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); + pmu->avoid_pmccntr |= + topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); if (!pmu->set_event_filter) pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c index e80f76d95e68..ca6bf8f86cfe 100644 --- a/drivers/perf/arm_pmu_acpi.c +++ b/drivers/perf/arm_pmu_acpi.c @@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, pmu_irqs); static int arm_pmu_acpi_register_irq(int cpu) { struct acpi_madt_generic_interrupt *gicc; - int gsi, trigger; + int gsi; gicc = acpi_cpu_get_madt_gicc(cpu); @@ -38,11 +38,6 @@ static int arm_pmu_acpi_register_irq(int cpu) if (!gsi) return 0; - if (gicc->flags & ACPI_MADT_PERFORMANCE_IRQ_MODE) - trigger = ACPI_EDGE_SENSITIVE; - else - trigger = ACPI_LEVEL_SENSITIVE; - /* * Helpfully, the MADT GICC doesn't have a polarity flag for the * "performance interrupt". Luckily, on compliant GICs the polarity is @@ -53,8 +48,12 @@ static int arm_pmu_acpi_register_irq(int cpu) * may not match the real polarity, but that should not matter. * * Other interrupt controllers are not supported with ACPI. + * + * The spec also indicates that the PMU interrupt can be edge + * triggered, which doesn't make any sense (SW needs to clear the + * interrupt condition for the level to drop). Ignore the silly flag. */ - return acpi_register_gsi(NULL, gsi, trigger, ACPI_ACTIVE_HIGH); + return acpi_register_gsi(NULL, gsi, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); } static void arm_pmu_acpi_unregister_irq(int cpu) diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 8014ff766cff..03359e078301 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -8,6 +8,7 @@ * This code is based heavily on the ARMv7 perf event code. */ +#include #include #include #include @@ -795,6 +796,7 @@ static void armv8pmu_disable_user_access(void) static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) { int i; + u64 userenr = ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_UEN; struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); if (is_pmuv3p9(cpu_pmu->pmuver)) { @@ -817,7 +819,10 @@ static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) } } - update_pmuserenr(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_UEN); + if (!cpu_pmu->avoid_pmccntr) + userenr |= ARMV8_PMU_USERENR_CR; + + update_pmuserenr(userenr); } static void armv8pmu_enable_event(struct perf_event *event) @@ -1002,13 +1007,7 @@ static bool armv8pmu_can_use_pmccntr(struct pmu_hw_events *cpuc, if (has_branch_stack(event)) return false; - /* - * The PMCCNTR_EL0 increments from the processor clock rather than - * the PE clock (ARM DDI0487 L.b D13.1.3) which means it'll continue - * counting on a WFI PE if one of its SMT sibling is not idle on a - * multi-threaded implementation. So don't use it on SMT cores. - */ - if (cpu_pmu->has_smt) + if (cpu_pmu->avoid_pmccntr) return false; return true; @@ -1250,7 +1249,8 @@ static int __armv8_pmuv3_map_event(struct perf_event *event, if (!(event->attach_state & PERF_ATTACH_TASK)) return -EINVAL; if (armv8pmu_event_is_64bit(event) && - (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) && + (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES || + armpmu->avoid_pmccntr) && !armv8pmu_has_long_event(armpmu)) return -EOPNOTSUPP; @@ -1299,6 +1299,45 @@ static int armv8_vulcan_map_event(struct perf_event *event) &armv8_vulcan_perf_cache_map); } +#ifdef CONFIG_ARM64 +/* + * List of CPUs that should avoid using PMCCNTR_EL0. + */ +static struct midr_range armv8pmu_avoid_pmccntr_cpus[] = { + /* + * NVIDIA Olympus may expose different WFI/WFE behaviour between the + * PMCCNTR_EL0 and the CPU_CYCLES event on programmable counters. + * While the CPU is in WFI/WFE state, the PMCCNTR_EL0 may still increment + * but the programmable counter may not. This is an implementation specific + * behavior and not an erratum. Perf assumes those two paths are + * interchangeable, so avoid using PMCCNTR_EL0 for CPU_CYCLES event. + * + * From ARM DDI0487 D14.4: + * It is IMPLEMENTATION SPECIFIC whether CPU_CYCLES and PMCCNTR count + * when the PE is in WFI or WFE state, even if the clocks are not stopped. + * + * From ARM DDI0487 D24.5.2: + * All counters are subject to any changes in clock frequency, including + * clock stopping caused by the WFI and WFE instructions. + * This means that it is CONSTRAINED UNPREDICTABLE whether or not + * PMCCNTR_EL0 continues to increment when clocks are stopped by WFI and + * WFE instructions. + */ + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + {} +}; + +static bool armv8pmu_is_in_avoid_pmccntr_cpus(void) +{ + return is_midr_in_range_list(armv8pmu_avoid_pmccntr_cpus); +} +#else +static bool armv8pmu_is_in_avoid_pmccntr_cpus(void) +{ + return false; +} +#endif + struct armv8pmu_probe_info { struct arm_pmu *pmu; bool present; @@ -1348,6 +1387,13 @@ static void __armv8pmu_probe_pmu(void *info) else cpu_pmu->reg_pmmir = 0; + /* + * On some CPUs, PMCCNTR_EL0 does not match the behavior of CPU_CYCLES + * programmable counter, so avoid routing cycles through PMCCNTR_EL0 to + * prevent inconsistency in the results. + */ + cpu_pmu->avoid_pmccntr |= armv8pmu_is_in_avoid_pmccntr_cpus(); + brbe_probe(cpu_pmu); } @@ -1361,7 +1407,7 @@ static int branch_records_alloc(struct arm_pmu *armpmu) struct pmu_hw_events *events_cpu; events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); - events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); + events_cpu->branch_stack = kzalloc(size, GFP_KERNEL); if (!events_cpu->branch_stack) return -ENOMEM; } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index dbd0da111639..b64cf2313a20 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -577,7 +577,7 @@ static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) * the page boundary following it. Keep the tail boundary if * that's lower. */ - if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); if (limit > head) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 68a54d97d2a8..bb32cd1084e9 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -77,6 +77,10 @@ #define CXL_PMU_GID_S2M_NDR 0x0024 #define CXL_PMU_GID_S2M_DRS 0x0025 #define CXL_PMU_GID_DDR 0x8000 +#define CXL_PMU_GID_QUEUE_OCC 0x8001 +#define CXL_PMU_GID_QUEUE_RESID 0x8002 +#define CXL_PMU_GID_RETRY_EVENTS 0x8003 +#define CXL_PMU_GID_THROTTLE 0x8004 static int cxl_pmu_cpuhp_state_num; @@ -106,6 +110,7 @@ struct cxl_pmu_info { int on_cpu; struct hlist_node node; bool filter_hdm; + bool filter_crb; int irq; }; @@ -142,6 +147,8 @@ static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; + info->filter_crb = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & + CXL_PMU_FILTER_CHAN_RANK_BANK; if (FIELD_GET(CXL_PMU_CAP_INT, val)) info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); else @@ -225,6 +232,8 @@ enum { cxl_pmu_edge_attr, cxl_pmu_hdm_filter_en_attr, cxl_pmu_hdm_attr, + cxl_pmu_crb_filter_en_attr, + cxl_pmu_crb_attr, }; static struct attribute *cxl_pmu_format_attr[] = { @@ -236,6 +245,8 @@ static struct attribute *cxl_pmu_format_attr[] = { [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), + [cxl_pmu_crb_filter_en_attr] = CXL_PMU_FORMAT_ATTR(crb_filter_en, "config1:19"), + [cxl_pmu_crb_attr] = CXL_PMU_FORMAT_ATTR(crb, "config2:32-63"), NULL }; @@ -246,7 +257,9 @@ static struct attribute *cxl_pmu_format_attr[] = { #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) +#define CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK BIT(19) #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) +#define CXL_PMU_ATTR_CONFIG2_CRB_MSK GENMASK_ULL(63, 32) static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, struct attribute *attr, int a) @@ -263,6 +276,11 @@ static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) return 0; + if (!info->filter_crb && + (attr == cxl_pmu_format_attr[cxl_pmu_crb_filter_en_attr] || + attr == cxl_pmu_format_attr[cxl_pmu_crb_attr])) + return 0; + return attr->mode; } @@ -319,6 +337,17 @@ static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); } +static u16 cxl_pmu_config1_crb_filter_en(struct perf_event *event) +{ + return FIELD_GET(CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK, + event->attr.config1); +} + +static u32 cxl_pmu_config2_get_crb(struct perf_event *event) +{ + return FIELD_GET(CXL_PMU_ATTR_CONFIG2_CRB_MSK, event->attr.config2); +} + static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -385,13 +414,23 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvtee, CXL_PMU_GID_M2S_REQ, BIT(7)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvptee, CXL_PMU_GID_M2S_REQ, BIT(11)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrdtee, CXL_PMU_GID_M2S_REQ, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_teupdate, CXL_PMU_GID_M2S_REQ, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevcttee, CXL_PMU_GID_M2S_REQ, BIT(14)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevctu, CXL_PMU_GID_M2S_REQ, BIT(15)), /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfill, CXL_PMU_GID_M2S_RWD, BIT(5)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrtee, CXL_PMU_GID_M2S_RWD, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptltee, CXL_PMU_GID_M2S_RWD, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfilltee, CXL_PMU_GID_M2S_RWD, BIT(13)), /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), @@ -406,15 +445,25 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curtee, CXL_PMU_GID_S2M_BISNP, BIT(8)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datatee, CXL_PMU_GID_S2M_BISNP, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invtee, CXL_PMU_GID_S2M_BISNP, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblktee, CXL_PMU_GID_S2M_BISNP, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datablktee, CXL_PMU_GID_S2M_BISNP, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblktee, CXL_PMU_GID_S2M_BISNP, BIT(14)), /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee, CXL_PMU_GID_S2M_NDR, BIT(5)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_s, CXL_PMU_GID_S2M_NDR, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_e, CXL_PMU_GID_S2M_NDR, BIT(7)), /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatatee, CXL_PMU_GID_S2M_DRS, BIT(2)), /* CXL rev 3.0 Table 13-5 directly lists these */ CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), @@ -423,6 +472,32 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), + /* CXL 4.0 Table 13-5 DDR add-on events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(ddr_cas_rd_ap, CXL_PMU_GID_DDR, BIT(7)), + CXL_PMU_EVENT_CXL_ATTR(ddr_cas_wr_ap, CXL_PMU_GID_DDR, BIT(8)), + CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_all_banks, CXL_PMU_GID_DDR, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_same_bank, CXL_PMU_GID_DDR, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_entry, CXL_PMU_GID_DDR, BIT(11)), + CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_exit, CXL_PMU_GID_DDR, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(ddr_rd_wr_ddr_bus_switching, CXL_PMU_GID_DDR, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_rd_req, CXL_PMU_GID_DDR, BIT(14)), + CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_wr_req, CXL_PMU_GID_DDR, BIT(15)), + /* CXL 4.0 Table 13-5 QUEUE OCCUPANCY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(rd_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(wr_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(rd_wr_merged_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(2)), + CXL_PMU_EVENT_CXL_ATTR(pwrdn_event, CXL_PMU_GID_QUEUE_OCC, BIT(3)), + /* CXL 4.0 Table 13-5 QUEUE RESIDENCY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(mc_rd_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(mc_wr_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(1)), + /* CXL 4.0 Table 13-5 RETRY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_rd_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_wr_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ca_parity, CXL_PMU_GID_RETRY_EVENTS, BIT(2)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ecc, CXL_PMU_GID_RETRY_EVENTS, BIT(3)), + /* CXL 4.0 Table 13-5 THROTTLE events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(thermal_throttle_event, CXL_PMU_GID_THROTTLE, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(power_throttle_event, CXL_PMU_GID_THROTTLE, BIT(1)), NULL }; @@ -571,6 +646,36 @@ static int cxl_pmu_event_init(struct perf_event *event) return -EOPNOTSUPP; /* TODO: Validation of any filter */ + if (cxl_pmu_config1_crb_filter_en(event)) { + if (!info->filter_crb) + return -EINVAL; + /* event group IDs are scoped by the CXL vendor ID */ + if (cxl_pmu_config_get_vid(event) != PCI_VENDOR_ID_CXL) + return -EINVAL; + + /* + * CRB filtering (Filter ID 1) is only valid for the DDR + * Interface, Queue Occupancy, Queue Residency and Retry + * event groups (CXL 4.0 Table 13-5). + */ + switch (cxl_pmu_config_get_gid(event)) { + case CXL_PMU_GID_DDR: + case CXL_PMU_GID_QUEUE_OCC: + case CXL_PMU_GID_QUEUE_RESID: + case CXL_PMU_GID_RETRY_EVENTS: + break; + default: + return -EINVAL; + } + + /* + * Filtering while counting multiple events is + * undefined behavior. + */ + if (hweight32(cxl_pmu_config_get_mask(event)) > 1) + return -EINVAL; + } + /* * Verify that it is possible to count what was requested. Either must * be a fixed counter that is a precise match or a configurable counter @@ -627,15 +732,23 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) hwc->state = 0; /* - * Currently only hdm filter control is implemented, this code will - * want generalizing when more filters are added. + * Filter ID=0: HDM decoder filter + * Filter ID=1: Channel/Rank/Bank (CRB) filter */ if (info->filter_hdm) { if (cxl_pmu_config1_hdm_filter_en(event)) cfg = cxl_pmu_config2_get_hdm_decoder(event); else cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ - writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + } + + if (info->filter_crb) { + if (cxl_pmu_config1_crb_filter_en(event)) + cfg = cxl_pmu_config2_get_crb(event); + else + cfg = GENMASK(31, 0); /* no filtering if 0xFFFF_FFFF */ + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 1)); } cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); diff --git a/drivers/perf/dwc_pcie_pmu.c b/drivers/perf/dwc_pcie_pmu.c index 5385401fa9cf..f5f3e5b83f5b 100644 --- a/drivers/perf/dwc_pcie_pmu.c +++ b/drivers/perf/dwc_pcie_pmu.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,7 @@ enum dwc_pcie_event_type { #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) +#define DWC_PCIE_PMU_TIMER_PERIOD_NS (2 * NSEC_PER_SEC) struct dwc_pcie_pmu { struct pmu pmu; @@ -93,6 +95,8 @@ struct dwc_pcie_pmu { /* Groups #6 and #7 */ DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); struct perf_event *time_based_event; + bool timer_enable; + struct hrtimer hrtimer; struct hlist_node cpuhp_node; int on_cpu; @@ -354,6 +358,26 @@ static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) return val; } +static void dwc_pcie_pmu_reset_time_based_counter(struct perf_event *event) +{ + struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + u64 prev; + + dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); + + /* + * The hardware counter is reset to zero when disabled. Synchronize + * prev_count so that the next event_update() computes the correct + * delta against the new counter baseline. + */ + do { + prev = local64_read(&hwc->prev_count); + } while (local64_cmpxchg(&hwc->prev_count, prev, 0) != prev); + + dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); +} + static void dwc_pcie_pmu_event_update(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -429,6 +453,26 @@ static int dwc_pcie_pmu_validate_group(struct perf_event *event) return 0; } +static enum hrtimer_restart dwc_pcie_pmu_hrtimer_callback(struct hrtimer *hrtimer) +{ + struct dwc_pcie_pmu *pcie_pmu = container_of(hrtimer, struct dwc_pcie_pmu, hrtimer); + struct perf_event *event = pcie_pmu->time_based_event; + struct hw_perf_event *hwc; + + if (!event) + return HRTIMER_NORESTART; + + hwc = &event->hw; + if (hwc->state & PERF_HES_STOPPED) + return HRTIMER_NORESTART; + + dwc_pcie_pmu_event_update(event); + dwc_pcie_pmu_reset_time_based_counter(event); + hrtimer_forward_now(hrtimer, ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS)); + + return HRTIMER_RESTART; +} + static int dwc_pcie_pmu_event_init(struct perf_event *event) { struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); @@ -478,10 +522,15 @@ static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) hwc->state = 0; local64_set(&hwc->prev_count, 0); - if (type == DWC_PCIE_LANE_EVENT) + if (type == DWC_PCIE_LANE_EVENT) { dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true); - else if (type == DWC_PCIE_TIME_BASE_EVENT) + } else if (type == DWC_PCIE_TIME_BASE_EVENT) { dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); + if (pcie_pmu->timer_enable) + hrtimer_start(&pcie_pmu->hrtimer, + ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS), + HRTIMER_MODE_REL_PINNED_HARD); + } } static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) @@ -495,11 +544,15 @@ static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) dwc_pcie_pmu_event_update(event); - if (type == DWC_PCIE_LANE_EVENT) + if (type == DWC_PCIE_LANE_EVENT) { dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false); - else if (type == DWC_PCIE_TIME_BASE_EVENT) + } else if (type == DWC_PCIE_TIME_BASE_EVENT) { dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); + if (pcie_pmu->timer_enable) + hrtimer_cancel(&pcie_pmu->hrtimer); + } + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } @@ -726,6 +779,16 @@ static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) pcie_pmu->ras_des_offset = vsec; pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); pcie_pmu->on_cpu = -1; + hrtimer_setup(&pcie_pmu->hrtimer, dwc_pcie_pmu_hrtimer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED_HARD); + + /* + * Use timer for updating time-based counts on platforms known + * to have narrowed counter. + */ + if (pdev->vendor == PCI_VENDOR_ID_PICOHEART) + pcie_pmu->timer_enable = true; + pcie_pmu->pmu = (struct pmu){ .name = name, .parent = &plat_dev->dev, diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index bcdf5575d71c..6d1e99abf110 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -858,10 +858,8 @@ static int ddr_perf_probe(struct platform_device *pdev) IRQF_NOBALANCING | IRQF_NO_THREAD, DDR_CPUHP_CB_NAME, pmu); - if (ret < 0) { - dev_err(&pdev->dev, "Request irq failed: %d", ret); + if (ret < 0) goto ddr_perf_err; - } pmu->irq = irq; ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); diff --git a/drivers/perf/fsl_imx9_ddr_perf.c b/drivers/perf/fsl_imx9_ddr_perf.c index 7050b48c0467..9dbc3187ef52 100644 --- a/drivers/perf/fsl_imx9_ddr_perf.c +++ b/drivers/perf/fsl_imx9_ddr_perf.c @@ -830,10 +830,8 @@ static int ddr_perf_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler, IRQF_NOBALANCING | IRQF_NO_THREAD, DDR_CPUHP_CB_NAME, pmu); - if (ret < 0) { - dev_err(&pdev->dev, "Request irq failed: %d", ret); + if (ret < 0) goto ddr_perf_err; - } pmu->irq = irq; ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); diff --git a/drivers/perf/fujitsu_uncore_pmu.c b/drivers/perf/fujitsu_uncore_pmu.c index aeeb68c66e1e..8c87d91e64b5 100644 --- a/drivers/perf/fujitsu_uncore_pmu.c +++ b/drivers/perf/fujitsu_uncore_pmu.c @@ -526,7 +526,7 @@ static int fujitsu_uncore_pmu_probe(struct platform_device *pdev) IRQF_NOBALANCING | IRQF_NO_THREAD, name, uncorepmu); if (ret) - return dev_err_probe(dev, ret, "Failed to request IRQ:%d\n", irq); + return ret; ret = irq_set_affinity(irq, cpumask_of(uncorepmu->cpu)); if (ret) diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c index f963e4f9e552..56a88fb0d3c2 100644 --- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c @@ -604,8 +604,7 @@ static int hisi_l3c_pmu_init_ext(struct hisi_pmu *l3c_pmu, struct platform_devic IRQF_NOBALANCING | IRQF_NO_THREAD, irqname, l3c_pmu); if (ret < 0) - return dev_err_probe(&pdev->dev, ret, - "Fail to request EXT IRQ: %d.\n", irq); + return ret; hisi_l3c_pmu->ext_irq[i] = irq; } diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c index de71dcf11653..f6f5b4470efe 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c @@ -192,11 +192,8 @@ int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu, ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(&pdev->dev), hisi_pmu); - if (ret < 0) { - dev_err(&pdev->dev, - "Fail to request IRQ: %d ret: %d.\n", irq, ret); + if (ret < 0) return ret; - } hisi_pmu->irq = irq; diff --git a/drivers/perf/marvell_cn10k_ddr_pmu.c b/drivers/perf/marvell_cn10k_ddr_pmu.c index 72ac17efd846..9c82bf4ee2c4 100644 --- a/drivers/perf/marvell_cn10k_ddr_pmu.c +++ b/drivers/perf/marvell_cn10k_ddr_pmu.c @@ -13,31 +13,43 @@ #include #include #include +#include + +/* SoC variant flags for struct ddr_pmu_platform_data (mutually exclusive in pdata) */ +#define IS_CN10K BIT(0) +#define IS_ODY BIT(1) +#define IS_CN20K BIT(2) /* Performance Counters Operating Mode Control Registers */ #define CN10K_DDRC_PERF_CNT_OP_MODE_CTRL 0x8020 #define ODY_DDRC_PERF_CNT_OP_MODE_CTRL 0x20020 +#define CN20K_DDRC_PERF_CNT_OP_MODE_CTRL 0x20000 #define OP_MODE_CTRL_VAL_MANUAL 0x1 /* Performance Counters Start Operation Control Registers */ #define CN10K_DDRC_PERF_CNT_START_OP_CTRL 0x8028 #define ODY_DDRC_PERF_CNT_START_OP_CTRL 0x200A0 +#define CN20K_DDRC_PERF_CNT_START_OP_CTRL 0x20080 #define START_OP_CTRL_VAL_START 0x1ULL #define START_OP_CTRL_VAL_ACTIVE 0x2 /* Performance Counters End Operation Control Registers */ #define CN10K_DDRC_PERF_CNT_END_OP_CTRL 0x8030 #define ODY_DDRC_PERF_CNT_END_OP_CTRL 0x200E0 +#define CN20K_DDRC_PERF_CNT_END_OP_CTRL 0x200C0 #define END_OP_CTRL_VAL_END 0x1ULL /* Performance Counters End Status Registers */ #define CN10K_DDRC_PERF_CNT_END_STATUS 0x8038 #define ODY_DDRC_PERF_CNT_END_STATUS 0x20120 +#define CN20K_DDRC_PERF_CNT_END_STATUS 0x20100 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1 /* Performance Counters Configuration Registers */ #define CN10K_DDRC_PERF_CFG_BASE 0x8040 #define ODY_DDRC_PERF_CFG_BASE 0x20160 +#define CN20K_DDRC_PERF_CFG_BASE 0x20140 +#define CN20K_DDRC_PERF_CFG1_BASE 0x20180 /* 8 Generic event counter + 2 fixed event counters */ #define DDRC_PERF_NUM_GEN_COUNTERS 8 @@ -61,6 +73,24 @@ * DO NOT change these event-id numbers, they are used to * program event bitmap in h/w. */ + +/* CN20K specific events */ +#define EVENT_PERF_OP_IS_RD16 61 +#define EVENT_PERF_OP_IS_RD32 60 +#define EVENT_PERF_OP_IS_WR16 59 +#define EVENT_PERF_OP_IS_WR32 58 +#define EVENT_OP_IS_ENTER_DSM 44 +#define EVENT_OP_IS_RFM 43 + + +#define EVENT_CN20K_OP_IS_ZQLATCH 62 +#define EVENT_CN20K_OP_IS_ZQSTART 63 +#define EVENT_CN20K_OP_IS_TCR_MRR 50 +#define EVENT_CN20K_OP_IS_DQSOSC_MRR 49 +#define EVENT_CN20K_OP_IS_DQSOSC_MPC 48 +#define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR 47 +#define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD 46 + #define EVENT_DFI_CMD_IS_RETRY 61 #define EVENT_RD_UC_ECC_ERROR 60 #define EVENT_RD_CRC_ERROR 59 @@ -87,6 +117,9 @@ #define EVENT_OP_IS_SPEC_REF 41 #define EVENT_OP_IS_CRIT_REF 40 #define EVENT_OP_IS_REFRESH 39 +#define EVENT_OP_IS_CAS_WCK_SUS 38 +#define EVENT_OP_IS_CAS_WS_OFF 37 +#define EVENT_OP_IS_CAS_WS 36 #define EVENT_OP_IS_ENTER_MPSM 35 #define EVENT_OP_IS_ENTER_POWERDOWN 31 #define EVENT_OP_IS_ENTER_SELFREF 27 @@ -183,8 +216,8 @@ struct ddr_pmu_platform_data { u64 cnt_freerun_clr; u64 cnt_value_wr_op; u64 cnt_value_rd_op; - bool is_cn10k; - bool is_ody; + u64 cfg1_base; + unsigned int silicon_flags; /* IS_CN10K, IS_ODY, or IS_CN20K */ }; static ssize_t cn10k_ddr_pmu_event_show(struct device *dev, @@ -336,6 +369,80 @@ static struct attribute *odyssey_ddr_perf_events_attrs[] = { NULL }; +static struct attribute *cn20k_ddr_perf_events_attrs[] = { + /* Programmable */ + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access, + EVENT_DFI_WR_DATA_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access, + EVENT_DFI_RD_DATA_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access, + EVENT_HPR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access, + EVENT_LPR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access, + EVENT_WR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, + EVENT_OP_IS_RD_OR_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, + EVENT_OP_IS_RD_ACTIVATE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, + EVENT_PRECHARGE_FOR_RDWR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other, + EVENT_PRECHARGE_FOR_OTHER), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, + EVENT_OP_IS_ENTER_POWERDOWN), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws, EVENT_OP_IS_CAS_WS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws_off, EVENT_OP_IS_CAS_WS_OFF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_wck_sus, EVENT_OP_IS_CAS_WCK_SUS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH), + CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rfm, EVENT_OP_IS_RFM), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_dsm, EVENT_OP_IS_ENTER_DSM), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd, + EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr, + EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_CN20K_OP_IS_DQSOSC_MPC), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_CN20K_OP_IS_DQSOSC_MRR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_CN20K_OP_IS_TCR_MRR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_CN20K_OP_IS_ZQSTART), + CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_CN20K_OP_IS_ZQLATCH), + CN10K_DDR_PMU_EVENT_ATTR(ddr_read16, EVENT_PERF_OP_IS_RD16), + CN10K_DDR_PMU_EVENT_ATTR(ddr_read32, EVENT_PERF_OP_IS_RD32), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write16, EVENT_PERF_OP_IS_WR16), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write32, EVENT_PERF_OP_IS_WR32), + /* Free run event counters */ + CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES), + NULL +}; + +static struct attribute_group cn20k_ddr_perf_events_attr_group = { + .name = "events", + .attrs = cn20k_ddr_perf_events_attrs, +}; + static struct attribute_group odyssey_ddr_perf_events_attr_group = { .name = "events", .attrs = odyssey_ddr_perf_events_attrs, @@ -393,6 +500,13 @@ static const struct attribute_group *odyssey_attr_groups[] = { NULL }; +static const struct attribute_group *cn20k_attr_groups[] = { + &cn20k_ddr_perf_events_attr_group, + &cn10k_ddr_perf_format_attr_group, + &cn10k_ddr_perf_cpumask_attr_group, + NULL +}; + /* Default poll timeout is 100 sec, which is very sufficient for * 48 bit counter incremented max at 5.6 GT/s, which may take many * hours to overflow. @@ -411,14 +525,38 @@ static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap, int err = 0; switch (eventid) { + case EVENT_CN20K_OP_IS_ZQLATCH ... EVENT_CN20K_OP_IS_ZQSTART: + if (ddr_pmu->p_data->silicon_flags & IS_CN20K) { + *event_bitmap = (1ULL << (eventid - 42)); + break; + } + err = -EINVAL; + break; case EVENT_DFI_PARITY_POISON ...EVENT_DFI_CMD_IS_RETRY: - if (!ddr_pmu->p_data->is_ody) { + /* + * 58..61: CN20K perf width events share numeric IDs with Odyssey + * DFI events; same 1ULL << (eventid - 1) bitmap on both paths. + */ + if (eventid >= EVENT_PERF_OP_IS_WR32 && + eventid <= EVENT_PERF_OP_IS_RD16) { + if (ddr_pmu->p_data->silicon_flags & IS_CN20K) { + *event_bitmap = (1ULL << (eventid - 1)); + break; + } + if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) { + err = -EINVAL; + break; + } + *event_bitmap = (1ULL << (eventid - 1)); + break; + } + if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) { err = -EINVAL; break; } fallthrough; case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD: - case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH: + case EVENT_OP_IS_CAS_WS ... EVENT_OP_IS_ZQLATCH: *event_bitmap = (1ULL << (eventid - 1)); break; case EVENT_OP_IS_ENTER_SELFREF: @@ -524,9 +662,9 @@ static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, int counter, bool enable) { const struct ddr_pmu_platform_data *p_data = pmu->p_data; + unsigned int silicon_flags = pmu->p_data->silicon_flags; u64 ctrl_reg = pmu->p_data->cnt_op_mode_ctrl; const struct ddr_pmu_ops *ops = pmu->ops; - bool is_ody = pmu->p_data->is_ody; u32 reg; u64 val; @@ -546,7 +684,7 @@ static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, writeq_relaxed(val, pmu->base + reg); - if (is_ody) { + if ((silicon_flags & IS_ODY) || (silicon_flags & IS_CN20K)) { if (enable) { /* * Setup the PMU counter to work in @@ -621,6 +759,7 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) { struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); const struct ddr_pmu_platform_data *p_data = pmu->p_data; + unsigned int silicon_flags = pmu->p_data->silicon_flags; const struct ddr_pmu_ops *ops = pmu->ops; struct hw_perf_event *hwc = &event->hw; u8 config = event->attr.config; @@ -642,10 +781,27 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { /* Generic counters, configure event id */ reg_offset = DDRC_PERF_CFG(p_data->cfg_base, counter); + ret = ddr_perf_get_event_bitmap(config, &val, pmu); if (ret) - return ret; + goto err_free_counter; + if (silicon_flags & IS_CN20K) { + if (config == EVENT_CN20K_OP_IS_ZQSTART || + config == EVENT_CN20K_OP_IS_ZQLATCH) { + /* ZQ lives in CFG1; clear stale event mask in CFG0 */ + writeq_relaxed(0, pmu->base + + DDRC_PERF_CFG(p_data->cfg_base, + counter)); + reg_offset = DDRC_PERF_CFG(p_data->cfg1_base, + counter); + } else { + /* Clear CFG1 so a prior ZQ select cannot linger */ + writeq_relaxed(0, pmu->base + + DDRC_PERF_CFG(p_data->cfg1_base, + counter)); + } + } writeq_relaxed(val, pmu->base + reg_offset); } else { /* fixed event counter, clear counter value */ @@ -661,6 +817,14 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) cn10k_ddr_perf_event_start(event, flags); return 0; + +err_free_counter: + if (pmu->active_events == 1) + hrtimer_cancel(&pmu->hrtimer); + pmu->active_events--; + cn10k_ddr_perf_free_counter(pmu, counter); + hwc->idx = -1; + return ret; } static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags) @@ -952,7 +1116,25 @@ static const struct ddr_pmu_platform_data cn10k_ddr_pmu_pdata = { .cnt_freerun_clr = 0, .cnt_value_wr_op = CN10K_DDRC_PERF_CNT_VALUE_WR_OP, .cnt_value_rd_op = CN10K_DDRC_PERF_CNT_VALUE_RD_OP, - .is_cn10k = TRUE, + .silicon_flags = IS_CN10K, +}; + +static const struct ddr_pmu_platform_data cn20k_ddr_pmu_pdata = { + .counter_overflow_val = 0, + .counter_max_val = GENMASK_ULL(63, 0), + .cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE, + .cfg_base = CN20K_DDRC_PERF_CFG_BASE, + .cfg1_base = CN20K_DDRC_PERF_CFG1_BASE, + .cnt_op_mode_ctrl = CN20K_DDRC_PERF_CNT_OP_MODE_CTRL, + .cnt_start_op_ctrl = CN20K_DDRC_PERF_CNT_START_OP_CTRL, + .cnt_end_op_ctrl = CN20K_DDRC_PERF_CNT_END_OP_CTRL, + .cnt_end_status = CN20K_DDRC_PERF_CNT_END_STATUS, + .cnt_freerun_en = 0, + .cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL, + .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR, + .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP, + .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP, + .silicon_flags = IS_CN20K, }; #endif @@ -979,7 +1161,7 @@ static const struct ddr_pmu_platform_data odyssey_ddr_pmu_pdata = { .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR, .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP, .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP, - .is_ody = TRUE, + .silicon_flags = IS_ODY, }; #endif @@ -989,8 +1171,7 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) struct cn10k_ddr_pmu *ddr_pmu; struct resource *res; void __iomem *base; - bool is_cn10k; - bool is_ody; + unsigned int silicon_flags; char *name; int ret; @@ -1014,10 +1195,9 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) ddr_pmu->base = base; ddr_pmu->p_data = dev_data; - is_cn10k = ddr_pmu->p_data->is_cn10k; - is_ody = ddr_pmu->p_data->is_ody; + silicon_flags = ddr_pmu->p_data->silicon_flags; - if (is_cn10k) { + if (silicon_flags & IS_CN10K) { ddr_pmu->ops = &ddr_pmu_ops; /* Setup the PMU counter to work in manual mode */ writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL, ddr_pmu->base + @@ -1039,7 +1219,7 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) }; } - if (is_ody) { + if (silicon_flags & IS_ODY) { ddr_pmu->ops = &ddr_pmu_ody_ops; ddr_pmu->pmu = (struct pmu) { @@ -1056,6 +1236,22 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) }; } + if (silicon_flags & IS_CN20K) { + ddr_pmu->ops = &ddr_pmu_ody_ops; + + ddr_pmu->pmu = (struct pmu) { + .module = THIS_MODULE, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + .task_ctx_nr = perf_invalid_context, + .attr_groups = cn20k_attr_groups, + .event_init = cn10k_ddr_perf_event_init, + .add = cn10k_ddr_perf_event_add, + .del = cn10k_ddr_perf_event_del, + .start = cn10k_ddr_perf_event_start, + .stop = cn10k_ddr_perf_event_stop, + .read = cn10k_ddr_perf_event_update, + }; + } /* Choose this cpu to collect perf data */ ddr_pmu->cpu = raw_smp_processor_id(); @@ -1088,6 +1284,12 @@ static void cn10k_ddr_perf_remove(struct platform_device *pdev) { struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); + /* + * Cancel the poll timer before further teardown so the handler + * cannot run after this function returns. + */ + hrtimer_cancel(&ddr_pmu->hrtimer); + cpuhp_state_remove_instance_nocalls( CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, &ddr_pmu->node); @@ -1098,6 +1300,7 @@ static void cn10k_ddr_perf_remove(struct platform_device *pdev) #ifdef CONFIG_OF static const struct of_device_id cn10k_ddr_pmu_of_match[] = { { .compatible = "marvell,cn10k-ddr-pmu", .data = &cn10k_ddr_pmu_pdata }, + { .compatible = "marvell,cn20k-ddr-pmu", .data = &cn20k_ddr_pmu_pdata }, { }, }; MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); @@ -1107,6 +1310,7 @@ MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = { {"MRVL000A", (kernel_ulong_t)&cn10k_ddr_pmu_pdata }, {"MRVL000C", (kernel_ulong_t)&odyssey_ddr_pmu_pdata}, + {"MRVL000B", (kernel_ulong_t)&cn20k_ddr_pmu_pdata}, {}, }; MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match); diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index ea8c85729937..57a28ddb9b0e 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -869,11 +869,8 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN, "l2-cache-pmu", cluster); - if (err) { - dev_err(&pdev->dev, - "Unable to request IRQ%d for L2 PMU counters\n", irq); + if (err) return err; - } dev_info(&pdev->dev, "Registered L2 cache PMU cluster %lld\n", fw_cluster_id); diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c index 66e6cabd6fff..5897ec5560fc 100644 --- a/drivers/perf/qcom_l3_pmu.c +++ b/drivers/perf/qcom_l3_pmu.c @@ -767,11 +767,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, ret, qcom_l3_cache__handle_irq, 0, name, l3pmu); - if (ret) { - dev_err(&pdev->dev, "Request for IRQ failed for slice @%pa\n", - &memrc->start); + if (ret) return ret; - } /* Add this instance to the list used by the offline callback */ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, &l3pmu->node); diff --git a/drivers/perf/starfive_starlink_pmu.c b/drivers/perf/starfive_starlink_pmu.c index b1c7dc4869bd..57b73575c43f 100644 --- a/drivers/perf/starfive_starlink_pmu.c +++ b/drivers/perf/starfive_starlink_pmu.c @@ -435,7 +435,7 @@ static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu, ret = devm_request_irq(&pdev->dev, irq, starlink_pmu_handle_irq, 0, STARLINK_PMU_PDEV_NAME, starlink_pmu); if (ret) - return dev_err_probe(&pdev->dev, ret, "Failed to request IRQ\n"); + return ret; starlink_pmu->irq = irq; diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c index 33b5497bdc06..b79409c750e6 100644 --- a/drivers/perf/xgene_pmu.c +++ b/drivers/perf/xgene_pmu.c @@ -1876,10 +1876,8 @@ static int xgene_pmu_probe(struct platform_device *pdev) rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(&pdev->dev), xgene_pmu); - if (rc) { - dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); + if (rc) return rc; - } xgene_pmu->irq = irq; diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 2f09f4b78bd3..dd422c56fbb1 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1196,8 +1196,7 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type, { u64 overflow_val = __mpam_msmon_overflow_val(type); - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) overflow_val *= 64; return overflow_val; @@ -1293,8 +1292,7 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - m->type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) now *= 64; if (nrdy) @@ -2025,6 +2023,9 @@ static void mpam_msc_drv_remove(struct platform_device *pdev) { struct mpam_msc *msc = platform_get_drvdata(pdev); + if (!msc) + return; + mutex_lock(&mpam_list_lock); mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock); @@ -2138,6 +2139,7 @@ static int mpam_msc_drv_probe(struct platform_device *pdev) static struct platform_driver mpam_msc_driver = { .driver = { .name = "mpam_msc", + .suppress_bind_attrs = true, }, .probe = mpam_msc_drv_probe, .remove = mpam_msc_drv_remove, diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 04d1a59f02af..def0e3a65c23 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -409,7 +409,11 @@ struct mpam_resctrl_res { struct mpam_resctrl_mon { struct mpam_class *class; - /* per-class data that resctrl needs will live here */ + /* Array of allocated MBWU monitors, indexed by (closid, rmid). */ + int *mbwu_idx_to_mon; + + /* Array of assigned MBWU monitors, indexed by resctrl's cntr_id. */ + int *assigned_counters; }; static inline int mpam_alloc_csu_mon(struct mpam_class *class) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 226ff6f532fa..9d223057953a 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -119,28 +119,9 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d { } -void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 closid, u32 rmid, int cntr_id, - enum resctrl_event_id eventid) -{ -} - -void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - enum resctrl_event_id evtid, u32 rmid, u32 closid, - u32 cntr_id, bool assign) -{ -} - -int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 unused, u32 rmid, int cntr_id, - enum resctrl_event_id eventid, u64 *val) -{ - return -EOPNOTSUPP; -} - bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { - return false; + return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); } int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) @@ -185,6 +166,26 @@ static void resctrl_reset_task_closids(void) read_unlock(&tasklist_lock); } +static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[QOS_L3_MBM_TOTAL_EVENT_ID]; + + if (!mon->class) + return; + + if (!mon->assigned_counters) + return; + + l3->mon.num_mbm_cntrs = mon->class->props.num_mbwu_mon; + if (cdp_enabled) + l3->mon.num_mbm_cntrs /= 2; + + /* + * Continue as normal even if enabling cdp causes there to be + * zero counters. This avoids giving resctrl mixed messages. + */ +} + int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) { u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; @@ -244,6 +245,7 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); resctrl_reset_task_closids(); + mpam_resctrl_monitor_sync_abmc_vals(l3); for_each_possible_cpu(cpu) mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); @@ -454,6 +456,14 @@ static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_c /* Shift closid to account for CDP */ closid = resctrl_get_config_index(closid, cdp_type); + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + if (mon_idx == -1) + return -ENOENT; + } + if (irqs_disabled()) { /* Check if we can access this domain without an IPI */ return -EIO; @@ -526,6 +536,84 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, closid, rmid, val); } +/* MBWU counters when in ABMC mode */ +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int mon_idx, + enum resctrl_event_id eventid, u64 *val) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return -EINVAL; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return -EINVAL; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu, + USE_PRE_ALLOCATED, closid, rmid, val); +} + +static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid) +{ + struct mon_cfg cfg = { }; + + if (!mpam_is_enabled()) + return; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + } + + if (mon_idx == -1) + return; + cfg.mon = mon_idx; + mpam_msmon_reset_mbwu(mon_comp, &cfg); +} + +static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, u32 closid, u32 rmid) +{ + if (cdp_enabled) { + __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid); + __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid); + } else { + __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid); + } +} + +/* Reset an assigned counter */ +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid); +} + /* * The rmid realloc threshold should be for the smallest cache exposed to * resctrl. @@ -606,6 +694,19 @@ static bool cache_has_usable_csu(struct mpam_class *class) return true; } +static bool class_has_usable_mbwu(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) + return false; + + if (!cprops->num_mbwu_mon) + return false; + + return true; +} + /* * Calculate the worst-case percentage change from each implemented step * in the control. @@ -925,6 +1026,50 @@ static void mpam_resctrl_pick_mba(void) } } +static void __free_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + if (array[i] < 0) + continue; + + mpam_free_mbwu_mon(class, array[i]); + array[i] = -1; + } +} + +static int __alloc_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + int mbwu_mon = mpam_alloc_mbwu_mon(class); + + if (mbwu_mon < 0) { + __free_mbwu_mon(class, array, num_mbwu_mon); + return mbwu_mon; + } + array[i] = mbwu_mon; + } + + return 0; +} + +static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon) +{ + int err; + + int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon); + if (!array) + return ERR_PTR(-ENOMEM); + + memset(array, -1, num_mbwu_mon * sizeof(*array)); + + err = __alloc_mbwu_mon(class, array, num_mbwu_mon); + if (err) + return ERR_PTR(err); + return_ptr(array); +} + static void counter_update_class(enum resctrl_event_id evt_id, struct mpam_class *class) { @@ -983,9 +1128,69 @@ static void mpam_resctrl_pick_counters(void) break; } } + + if (class_has_usable_mbwu(class) && + topology_matches_l3(class) && + traffic_matches_l3(class)) { + pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n", + class->level); + + /* + * An MSC measures bandwidth for a path determined by + * its location in hardware. We can't distinguish + * traffic by destination so we don't know if it's + * staying on the same NUMA node. Hence, we can't + * calculate mbm_local except when we only have one L3 + * and it's equivalent to mbm_total and so always use + * mbm_total. + */ + counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class); + } } } +static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, + bool assign) +{ + /* Same CDP index remap as closid; maps cntr_id to assigned_counters[]. */ + u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type); + + closid = resctrl_get_config_index(closid, cdp_type); + mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + if (assign) + mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx]; + else + mon->mbwu_idx_to_mon[mbwu_idx] = -1; +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (evtid != QOS_L3_MBM_TOTAL_EVENT_ID) { + pr_debug("unexpected event id\n"); + return; + } + + if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) { + pr_debug("monitor arrays not allocated\n"); + return; + } + + if (cdp_enabled) { + __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign); + __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign); + } else { + __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign); + } + + resctrl_arch_reset_cntr(r, d, closid, rmid, cntr_id, QOS_L3_MBM_TOTAL_EVENT_ID); +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { struct mpam_class *class = res->class; @@ -1063,6 +1268,43 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +/* + * This must run after all event counters have been picked so that any free + * running counters have already been allocated. + */ +static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + size_t num_rmid = resctrl_arch_system_num_rmid_idx(); + struct rdt_resource *l3 = &res->resctrl_res; + struct mpam_class *class = mon->class; + u16 num_mbwu_mon; + int *cntrs; + + int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid); + if (!rmid_array) { + pr_debug("Failed to allocate RMID array\n"); + return -ENOMEM; + } + memset(rmid_array, -1, num_rmid * sizeof(*rmid_array)); + + num_mbwu_mon = class->props.num_mbwu_mon; + cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon); + if (IS_ERR(cntrs)) + return PTR_ERR(cntrs); + mon->assigned_counters = cntrs; + mon->mbwu_idx_to_mon = no_free_ptr(rmid_array); + + l3->mon.mbm_cntr_assignable = true; + l3->mon.mbm_assign_on_mkdir = true; + l3->mon.mbm_cntr_configurable = false; + l3->mon.mbm_cntr_assign_fixed = true; + + mpam_resctrl_monitor_sync_abmc_vals(l3); + + return 0; +} + static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, enum resctrl_event_id type) { @@ -1107,8 +1349,21 @@ static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, */ l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); - if (resctrl_enable_mon_event(type, false, 0, NULL)) - l3->mon_capable = true; + if (type == QOS_L3_MBM_TOTAL_EVENT_ID) { + int err; + + err = mpam_resctrl_monitor_init_abmc(mon); + if (err) + return err; + + static_assert(MAX_EVT_CONFIG_BITS == 0x7f); + l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS; + } + + if (!resctrl_enable_mon_event(type, false, 0, NULL)) + return -EINVAL; + + l3->mon_capable = true; return 0; } @@ -1671,6 +1926,23 @@ void mpam_resctrl_exit(void) resctrl_exit(); } +static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class) +{ + u32 num_mbwu_mon = class->props.num_mbwu_mon; + + if (!mon->mbwu_idx_to_mon) + return; + + if (mon->assigned_counters) { + __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon); + kvfree(mon->assigned_counters); + mon->assigned_counters = NULL; + } + + kvfree(mon->mbwu_idx_to_mon); + mon->mbwu_idx_to_mon = NULL; +} + /* * The driver is detaching an MSC from this class, if resctrl was using it, * pull on resctrl_exit(). @@ -1693,6 +1965,8 @@ void mpam_resctrl_teardown_class(struct mpam_class *class) for_each_mpam_resctrl_mon(mon, eventid) { if (mon->class == class) { mon->class = NULL; + + mpam_resctrl_teardown_mon(mon, class); break; } } diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 32cd038cb79b..0eeddd1ff05b 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -16,54 +16,38 @@ /** * struct arm_cca_token_info - a descriptor for the token buffer. - * @challenge: Pointer to the challenge data - * @challenge_size: Size of the challenge data * @granule: PA of the granule to which the token will be written * @offset: Offset within granule to start of buffer in bytes - * @result: result of rsi_attestation_token_continue operation */ struct arm_cca_token_info { - void *challenge; - unsigned long challenge_size; phys_addr_t granule; unsigned long offset; - unsigned long result; }; -static void arm_cca_attestation_init(void *param) -{ - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; - - info->result = rsi_attestation_token_init(info->challenge, - info->challenge_size); -} - /** * arm_cca_attestation_continue - Retrieve the attestation token data. * - * @param: pointer to the arm_cca_token_info + * @info: pointer to the arm_cca_token_info * * Attestation token generation is a long running operation and therefore * the token data may not be retrieved in a single call. Moreover, the * token retrieval operation must be requested on the same CPU on which the * attestation token generation was initialised. - * This helper function is therefore scheduled on the same CPU multiple + * This helper function must therefore be executed on the same CPU multiple * times until the entire token data is retrieved. */ -static void arm_cca_attestation_continue(void *param) +static unsigned long +arm_cca_attestation_continue(struct arm_cca_token_info *info) { + unsigned long ret; unsigned long len; unsigned long size; - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; size = RSI_GRANULE_SIZE - info->offset; - info->result = rsi_attestation_token_continue(info->granule, - info->offset, size, &len); + ret = rsi_attestation_token_continue(info->granule, info->offset, size, + &len); info->offset += len; + return ret; } /** @@ -74,8 +58,8 @@ static void arm_cca_attestation_continue(void *param) * * Initialise the attestation token generation using the challenge data * passed in the TSM descriptor. Allocate memory for the attestation token - * and schedule calls to retrieve the attestation token on the same CPU - * on which the attestation token generation was initialised. + * and retrieve the attestation token on the same CPU on which the + * attestation token generation was initialised. * * The challenge data must be at least 32 bytes and no more than 64 bytes. If * less than 64 bytes are provided it will be zero padded to 64 bytes. @@ -85,12 +69,11 @@ static void arm_cca_attestation_continue(void *param) * * %-EINVAL - A parameter was not valid. * * %-ENOMEM - Out of memory. * * %-EFAULT - Failed to get IPA for memory page(s). - * * A negative status code as returned by smp_call_function_single(). */ static int arm_cca_report_new(struct tsm_report *report, void *data) { - int ret; - int cpu; + int ret = 0; + unsigned long rsi_result; long max_size; unsigned long token_size = 0; struct arm_cca_token_info info; @@ -103,37 +86,33 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) /* * The attestation token 'init' and 'continue' calls must be - * performed on the same CPU. smp_call_function_single() is used - * instead of simply calling get_cpu() because of the need to - * allocate outblob based on the returned value from the 'init' - * call and that cannot be done in an atomic context. + * performed on the same CPU, so disable CPU migration around + * those operations. */ - cpu = smp_processor_id(); + migrate_disable(); - info.challenge = desc->inblob; - info.challenge_size = desc->inblob_len; - - ret = smp_call_function_single(cpu, arm_cca_attestation_init, - &info, true); - if (ret) - return ret; - max_size = info.result; - - if (max_size <= 0) - return -EINVAL; + max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len); + if (max_size <= 0) { + ret = -EINVAL; + goto exit_migrate_enable; + } /* Allocate outblob */ token = kvzalloc(max_size, GFP_KERNEL); - if (!token) - return -ENOMEM; + if (!token) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* * Since the outblob may not be physically contiguous, use a page * to bounce the buffer from RMM. */ buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; + if (!buf) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* Get the PA of the memory page(s) that were allocated */ info.granule = (unsigned long)virt_to_phys(buf); @@ -144,21 +123,15 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) info.offset = 0; do { /* - * Schedule a call to retrieve a sub-granule chunk - * of data per loop iteration. + * Retrieve a sub-granule chunk of data per loop + * iteration. */ - ret = smp_call_function_single(cpu, - arm_cca_attestation_continue, - (void *)&info, true); - if (ret != 0) { - token_size = 0; - goto exit_free_granule_page; - } - } while (info.result == RSI_INCOMPLETE && + rsi_result = arm_cca_attestation_continue(&info); + } while (rsi_result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); /* Break out in case of failure */ - if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { + if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; @@ -173,12 +146,14 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) break; memcpy(&token[token_size], buf, info.offset); token_size += info.offset; - } while (info.result == RSI_INCOMPLETE); + } while (rsi_result == RSI_INCOMPLETE); report->outblob = no_free_ptr(token); exit_free_granule_page: report->outblob_len = token_size; free_pages_exact(buf, RSI_GRANULE_SIZE); +exit_migrate_enable: + migrate_enable(); return ret; } @@ -223,7 +198,7 @@ module_exit(arm_cca_guest_exit); /* modalias, so userspace can autoload this module when RSI is available */ static const struct platform_device_id arm_cca_match[] __maybe_unused = { - { RSI_PDEV_NAME, 0}, + { .name = RSI_PDEV_NAME }, { } }; diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c index d66291def0f4..26fe9c3f22e3 100644 --- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c +++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c @@ -17,6 +17,7 @@ #include static size_t pkvm_granule; +DEFINE_STATIC_KEY_FALSE_RO(pkvm_guest); static int arm_smccc_do_one_page(u32 func_id, phys_addr_t phys) { @@ -120,4 +121,6 @@ void pkvm_init_hyp_services(void) if (kvm_arm_hyp_service_available(ARM_SMCCC_KVM_FUNC_MMIO_GUARD)) arm64_ioremap_prot_hook_register(&mmio_guard_ioremap_hook); + + static_branch_enable(&pkvm_guest); } diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 19830146c644..94a6277edab2 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -179,6 +179,8 @@ extern int cppc_get_transition_latency(int cpu); extern bool cpc_ffh_supported(void); extern bool cpc_supported_by_cpu(void); extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); +extern int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2); extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); @@ -254,6 +256,11 @@ static inline int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) { return -EOPNOTSUPP; } +static inline int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + return -EOPNOTSUPP; +} static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) { return -EOPNOTSUPP; diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index f652a5028b59..b9dc21c241be 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -37,6 +37,15 @@ int sdei_event_unregister(u32 event_num); int sdei_event_enable(u32 event_num); int sdei_event_disable(u32 event_num); +/* + * Signal the software-signalled event (event 0) to another PE, NMI-like. + * @mpidr is the target's MPIDR affinity. + */ +int sdei_event_signal(u32 event_num, u64 mpidr); + +/* Was SDEI firmware probed and usable? */ +bool sdei_is_present(void); + /* GHES register/unregister helpers */ int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb, sdei_event_callback *critical_cb); diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 1c9d40e09107..2c17239aacea 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2640,6 +2640,8 @@ #define PCI_VENDOR_ID_SUNIX 0x1fd4 #define PCI_DEVICE_ID_SUNIX_1999 0x1999 +#define PCI_VENDOR_ID_PICOHEART 0x20fa + #define PCI_VENDOR_ID_HINT 0x3388 #define PCI_DEVICE_ID_HINT_VXPROII_IDE 0x8013 diff --git a/include/linux/pcie-dwc.h b/include/linux/pcie-dwc.h index 8ff778e7aec0..d3cd701a58e3 100644 --- a/include/linux/pcie-dwc.h +++ b/include/linux/pcie-dwc.h @@ -26,6 +26,8 @@ static const struct dwc_pcie_vsec_id dwc_pcie_rasdes_vsec_ids[] = { .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_AMPERE, .vsec_id = 0x02, .vsec_rev = 0x4 }, + { .vendor_id = PCI_VENDOR_ID_PICOHEART, + .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_QCOM, .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_ROCKCHIP, diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 52b37f7bdbf9..02d2c7f45b52 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -119,7 +119,7 @@ struct arm_pmu { /* PMUv3 only */ int pmuver; - bool has_smt; + bool avoid_pmccntr; u64 reg_pmmir; u64 reg_brbidr; #define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 diff --git a/include/uapi/linux/arm_sdei.h b/include/uapi/linux/arm_sdei.h index af0630ba5437..22eb61612673 100644 --- a/include/uapi/linux/arm_sdei.h +++ b/include/uapi/linux/arm_sdei.h @@ -22,6 +22,7 @@ #define SDEI_1_0_FN_SDEI_PE_UNMASK SDEI_1_0_FN(0x0C) #define SDEI_1_0_FN_SDEI_INTERRUPT_BIND SDEI_1_0_FN(0x0D) #define SDEI_1_0_FN_SDEI_INTERRUPT_RELEASE SDEI_1_0_FN(0x0E) +#define SDEI_1_0_FN_SDEI_EVENT_SIGNAL SDEI_1_0_FN(0x0F) #define SDEI_1_0_FN_SDEI_PRIVATE_RESET SDEI_1_0_FN(0x11) #define SDEI_1_0_FN_SDEI_SHARED_RESET SDEI_1_0_FN(0x12) diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index bcc6df79301a..4ead4e72097c 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -6,6 +6,7 @@ #define __TOOLS_LINUX_FILTER_H #include +#include /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function diff --git a/tools/include/linux/types.h b/tools/include/linux/types.h index d41f8a261bce..b6c473b7920d 100644 --- a/tools/include/linux/types.h +++ b/tools/include/linux/types.h @@ -23,6 +23,11 @@ typedef enum { __GFP_HIGH } gfp_t; +#ifdef __SIZEOF_INT128__ +typedef __signed__ __int128 __s128 __attribute__((aligned(16))); +typedef unsigned __int128 __u128 __attribute__((aligned(16))); +#endif + /* * We define u64 as uint64_t for every architecture * so that we can print it with "%"PRIx64 without getting warnings. diff --git a/tools/testing/selftests/arm64/abi/.gitignore b/tools/testing/selftests/arm64/abi/.gitignore index 44f8b80f37e3..491a80db9dff 100644 --- a/tools/testing/selftests/arm64/abi/.gitignore +++ b/tools/testing/selftests/arm64/abi/.gitignore @@ -1,4 +1,6 @@ hwcap ptrace +seccomp_ptrace_x0_bypass +seccomp_ret_trace_x0_bypass syscall-abi tpidr2 diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile index 483488f8c2ad..e91d4cdf17ad 100644 --- a/tools/testing/selftests/arm64/abi/Makefile +++ b/tools/testing/selftests/arm64/abi/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 ARM Limited -TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 +CFLAGS += $(KHDR_INCLUDES) -I$(top_srcdir)/tools/include + +TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 seccomp_ptrace_x0_bypass seccomp_ret_trace_x0_bypass include ../../lib.mk diff --git a/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c new file mode 100644 index 000000000000..4ee8e5aaad6f --- /dev/null +++ b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test that seccomp, tracepoints and audit observe the correct syscall + * arguments after a ptracer has modified them at syscall-enter-stop. + * + * On arm64, both the first argument and the return value of a syscall + * are passed in register x0. The original x0 is saved in + * pt_regs::orig_x0 during syscall entry and returned as the first + * argument by syscall_get_arguments(). Because ptrace modifications + * to x0 are not automatically reflected in orig_x0, seccomp, tracepoints + * and audit may see a stale value unless orig_x0 is explicitly + * re-synchronised after a ptrace stop. + * + * This test sets up a seccomp filter that allows write(2, ...) but kills + * the task for any other fd. A ptracer changes the fd argument from 2 + * to 1 at the syscall-enter stop. If the orig_x0 re-sync works, seccomp + * sees the modified argument (fd=1) and kills the child with SIGSYS + * (test passes). If orig_x0 is not re-synced, seccomp sees the original + * fd=2, the write succeeds and the child exits normally (test fails, + * vulnerability present). + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#define EXPECTED_TESTS 1 + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define ARG0_OFFSET (offsetof(struct seccomp_data, args)) +#else +#define ARG0_OFFSET (offsetof(struct seccomp_data, args) + 4) +#endif + +static int do_child(void) +{ + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL)) + ksft_exit_fail_perror("PTRACE_TRACEME"); + + if (raise(SIGSTOP)) + ksft_exit_fail_perror("raise(SIGSTOP)"); + + /* + * Seccomp filter: + * If syscall is not write -> ALLOW + * If syscall is write: + * - If args[0] (fd) == 2 -> ALLOW + * - Otherwise -> KILL + */ + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), /* nr */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, ARG0_OFFSET), /* args[0] */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 2, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog prog = { + .len = ARRAY_SIZE(filter), + .filter = filter, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + ksft_exit_fail_perror("prctl NO_NEW_PRIVS"); + + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) + ksft_exit_fail_perror("prctl SECCOMP"); + + /* + * Invoke write(2, ...) while the tracer will change the first + * argument (fd) from 2 to 1 at syscall entry. + */ + syscall(__NR_write, 2, NULL, 0); + _exit(0); +} + +static int do_parent(pid_t child) +{ + bool bypass = false; + int status; + + /* Wait for the initial SIGSTOP */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid failed"); + + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + ksft_exit_fail_msg("unexpected stop status"); + + if (ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD | PTRACE_O_EXITKILL)) + ksft_exit_fail_perror("PTRACE_SETOPTIONS"); + + if (ptrace(PTRACE_SYSCALL, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + + while (1) { + int sig; + + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid lost child"); + + if (WIFEXITED(status)) { + /* Child exited normally – bypass succeeded */ + bypass = true; + break; + } + + if (WIFSIGNALED(status)) { + sig = WTERMSIG(status); + if (sig == SIGSYS) + break; + ksft_exit_fail_msg("child died unexpectedly from signal %d (%s)", + sig, strsignal(sig)); + } + + if (!WIFSTOPPED(status)) + ksft_exit_fail_msg("unexpected wait status"); + + sig = WSTOPSIG(status); + + if (sig == (SIGTRAP | 0x80)) { + struct user_regs_struct regs; + struct iovec iov = { + .iov_base = ®s, + .iov_len = sizeof(regs), + }; + + if (ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("PTRACE_GETREGSET"); + + unsigned long syscall_nr = regs.regs[8]; + unsigned long x0 = regs.regs[0]; + + /* Modify fd from 2 to 1 at write entry */ + if (syscall_nr == __NR_write && x0 == 2) { + regs.regs[0] = 1; + if (ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("PTRACE_SETREGSET"); + } + + if (ptrace(PTRACE_SYSCALL, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + } else { + /* Forward other signals */ + if (ptrace(PTRACE_SYSCALL, child, 0, sig)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + } + } + + /* bypass == true means vulnerability exists -> test fails */ + return bypass ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int main(void) +{ + pid_t child; + + ksft_print_header(); + ksft_set_plan(EXPECTED_TESTS); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork failed: %s", strerror(errno)); + + if (!child) + return do_child(); + + /* + * do_parent() returns EXIT_SUCCESS if the child was killed by + * SIGSYS (i.e. seccomp correctly saw the modified argument), + * and EXIT_FAILURE if the child exited normally (bypass). + */ + int result = do_parent(child); + + ksft_test_result(result == EXIT_SUCCESS, "seccomp_ptrace_x0_bypass\n"); + + ksft_print_cnts(); + return result; +} diff --git a/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c new file mode 100644 index 000000000000..a23081763328 --- /dev/null +++ b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test for SECCOMP_RET_TRACE argument modification bypass + * via stale orig_x0 during filter re-evaluation. + * + * On arm64, syscall_get_arguments() reads the first argument from + * regs->orig_x0. When a seccomp filter returns SECCOMP_RET_TRACE, + * ptrace may modify regs->regs[0] while orig_x0 remains unchanged. + * The kernel then re-evaluates the filter; if it sees the stale + * orig_x0, it may incorrectly allow a syscall that the tracer intended + * to block. + * + * This test installs a filter that: + * - TRACEs write() when fd == 2 + * - returns ERRNO(EPERM) when fd == 1 + * - allows all other syscalls + * + * The child calls write(2, ...). The parent catches the SECCOMP stop, + * changes x0 (fd) from 2 to 1, and resumes the child. + * + * If re-evaluation sees the old fd=2 (stale orig_x0), the filter + * returns TRACE again; because recheck_after_trace is true, the kernel + * allows the syscall to proceed. write(1, ...) succeeds, child exits 0. + * -> test FAIL (bypass detected). + * + * If re-evaluation sees the new fd=1 (synced orig_x0), the filter + * returns ERRNO(EPERM), write fails, child exits 1. + * -> test PASS (no bypass). + * + * No special privileges required beyond CAP_SYS_PTRACE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#define PTRACE_EVENT_MASK(status) ((status) >> 16) + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define ARG0_OFFSET (offsetof(struct seccomp_data, args)) +#else +#define ARG0_OFFSET (offsetof(struct seccomp_data, args) + 4) +#endif + +static int do_child(void) +{ + long ret; + + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL)) + _exit(2); + + raise(SIGSTOP); /* synchronize with parent */ + + /* + * Filter: + * if syscall == write: + * if fd == 2 -> TRACE + * if fd == 1 -> ERRNO(EPERM) + * else -> ALLOW + * else -> ALLOW + */ + struct sock_filter filter[] = { + /* Load syscall number */ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + /* If not write, allow */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 0, 5), + /* Load first argument (fd) */ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, ARG0_OFFSET), + /* fd == 2 ? */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 2, 0, 1), + /* Yes: TRACE */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRACE), + /* fd == 1 ? */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 1), + /* Yes: ERRNO(EPERM) */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA)), + /* Other fd: ALLOW */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + + struct sock_fprog prog = { + .len = ARRAY_SIZE(filter), + .filter = filter, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + _exit(3); + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) + _exit(4); + + /* + * write(2, ...) triggers TRACE, parent changes fd to 1. + * If re-eval sees fd=1 -> ERRNO -> write fails, ret = -EPERM. + * If re-eval sees fd=2 -> TRACE again -> allowed -> write succeeds. + */ + ret = syscall(__NR_write, 2, "", 0); + _exit(ret == 0 ? 0 : 1); +} + +int main(void) +{ + struct user_pt_regs regs; + struct iovec iov = { .iov_base = ®s, .iov_len = sizeof(regs) }; + pid_t child; + int status; + + ksft_print_header(); + ksft_set_plan(1); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork failed: %s", strerror(errno)); + + if (!child) + return do_child(); + + /* 1. Wait for initial SIGSTOP */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid SIGSTOP"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + ksft_exit_fail_msg("unexpected initial stop"); + + /* 2. Enable SECCOMP ptrace events */ + if (ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESECCOMP)) + ksft_exit_fail_msg("PTRACE_SETOPTIONS"); + + /* 3. Continue child to hit SECCOMP stop */ + if (ptrace(PTRACE_CONT, child, 0, 0)) + ksft_exit_fail_msg("PTRACE_CONT"); + + /* 4. Wait for SECCOMP stop */ + while (1) { + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid SECCOMP"); + if (WIFEXITED(status)) { + ksft_test_result_fail("child exited before SECCOMP stop\n"); + goto out; + } + if (WIFSIGNALED(status)) { + ksft_test_result_fail("child killed unexpectedly\n"); + goto out; + } + if (WIFSTOPPED(status) && + WSTOPSIG(status) == SIGTRAP && + PTRACE_EVENT_MASK(status) == PTRACE_EVENT_SECCOMP) + break; + ptrace(PTRACE_CONT, child, 0, WSTOPSIG(status)); + } + + /* 5. Modify x0 (fd) from 2 to 1 */ + if (ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("GETREGSET"); + if (regs.regs[8] != __NR_write || regs.regs[0] != 2) { + ksft_test_result_fail("unexpected regs: syscall=%llu, x0=%llu\n", + regs.regs[8], regs.regs[0]); + goto out; + } + regs.regs[0] = 1; + if (ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("SETREGSET"); + + /* 6. Resume child */ + if (ptrace(PTRACE_CONT, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_CONT"); + + /* 7. Reap child – must exit normally */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("final waitpid"); + + if (!WIFEXITED(status)) { + ksft_test_result_fail("child did not exit normally\n"); + goto out; + } + + if (WEXITSTATUS(status) != 0) + ksft_test_result_pass("seccomp correctly denied modified syscall\n"); + else + ksft_test_result_fail("write succeeded, orig_x0 bypass likely\n"); + +out: + if (child > 0) { + kill(child, SIGKILL); + waitpid(child, NULL, 0); + } + ksft_print_cnts(); + return ksft_get_fail_cnt() ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 22c584b78be5..b435837c8c0e 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 80e072f221cd..7ef7835389e7 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -298,15 +298,20 @@ function irritator_handler add x0, x0, #1 str x0, [x2, #ucontext_regs + 8 * 23] +#ifndef SSVE // Corrupt some random Z-regs movi v0.8b, #1 movi v9.16b, #2 movi v31.8b, #3 // And P0 ptrue p0.d -#ifndef SSVE // And FFR wrffr p15.b +#else + // Enter and exit streaming mode, will reset all of the V, Z, P + // and FFR registers that the system has. + smstart_sm + smstop #endif ret