diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..757110421543 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -701,12 +701,34 @@ config ARM64_ERRATUM_1530923 If unsure, say Y. -config ARM64_WORKAROUND_REPEAT_TLBI +config ARM64_WORKAROUND_REPEAT_TLBI_SYNC bool + # This workaround is (only) suitable for TLB invalidation errata where + # all of the following conditions are true: + # + # - The effects of the errata are only a loss of ordering/completion + # for explicit memory accesses when the TLBI is completed with a DSB. + # The removal of TLB entries is not affected. + # + # Note that architecturally, S2-only invalidation does not remove + # combined S1+S2 entries, and does not complete accesses translated + # via those S1+S2 entries. Consequently, where this condition holds, + # the errata do not affect S2-only invalidation. + # + # - The errata only affect broadcast TLB invalidation operations (e.g. + # TLBI VMALLE1IS), and do not affect local TLB invalidation + # operations (e.g. TLBI VMALLE1). + # + # - After any number of affected TLBI operations are completed with a + # DSB, the errata can be mitigated by executing a single arbitrary + # broadcast TLBI (which targets an arbitrary translation regime), + # followed by a DSB. + # + # For more rationale, see commit a8f78680ee6bf795. config ARM64_ERRATUM_2441007 bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A55 erratum #2441007. @@ -722,7 +744,7 @@ config ARM64_ERRATUM_2441007 config ARM64_ERRATUM_1286807 bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A76 erratum 1286807. @@ -944,7 +966,7 @@ config ARM64_ERRATUM_2224489 config ARM64_ERRATUM_2441009 bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A510 erratum #2441009. @@ -1156,7 +1178,7 @@ config ARM64_ERRATUM_4193714 config ARM64_ERRATUM_4118414 bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for the following errata: @@ -1340,7 +1362,7 @@ config QCOM_FALKOR_ERRATUM_1003 config QCOM_FALKOR_ERRATUM_1009 bool "Falkor E1009: Prematurely complete a DSB after a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help On Falkor v1, the CPU may prematurely complete a DSB following a TLBI xxIS invalidate maintenance operation. Repeat the TLBI operation diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 25c61cda901c..76350b38f0d7 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -60,8 +60,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); case ARM64_WORKAROUND_DISABLE_CNP: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); - case ARM64_WORKAROUND_REPEAT_TLBI: - return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); + case ARM64_WORKAROUND_REPEAT_TLBI_SYNC: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); case ARM64_WORKAROUND_4193714: diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index e0e84332f51b..14a78ac0f800 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -236,7 +236,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) #define __repeat_tlbi_sync(op, arg...) \ do { \ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI_SYNC)) \ break; \ __tlbi(op, ##arg); \ dsb(ish); \ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648..685077d44ad1 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -309,7 +309,7 @@ static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *_ sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56)); } -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009 { @@ -733,10 +733,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .match_list = qcom_erratum_1003_list, }, #endif -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC { .desc = "Broken broadcast TLBI completion", - .capability = ARM64_WORKAROUND_REPEAT_TLBI, + .capability = ARM64_WORKAROUND_REPEAT_TLBI_SYNC, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, .match_list = arm64_repeat_tlbi_list, diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..f8368e5d81a8 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -124,7 +124,7 @@ WORKAROUND_DISABLE_CNP WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF -WORKAROUND_REPEAT_TLBI +WORKAROUND_REPEAT_TLBI_SYNC WORKAROUND_SPECULATIVE_AT WORKAROUND_SPECULATIVE_SSBS WORKAROUND_SPECULATIVE_UNPRIV_LOAD