github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 17:47:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

arm64: Avoid eager DVMSync reclaim batches with C1-Pro SME erratum

Browse Source 
The C1-Pro SME DVMSync workaround currently samples mm_cpumask() from
arch_tlbbatch_add_pending(). It requires a DSB after every batched TLBI
so that the mask read is ordered after the hardware DVMSync, defeating
much of the reclaim batching benefit.

Introduce the sme_active_cpus mask tracking which CPUs run in user-space
with SME enabled and use it for batch flushing instead of accumulating
the mm_cpumask() of the unmapped pages.

Fixes: 0baba94a97 ("arm64: errata: Work around early CME DVMSync acknowledgement")
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Tested-by: Joshua Liu <josliu@google.com>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Catalin Marinas 2026-06-10 11:37:16 +01:00 committed by Will Deacon
parent f9a82544c7
commit 534eb6940a
4 changed files with 17 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
arch/arm64/include/asm/tlbbatch.h
View File

@ -2,17 +2,11 @@
#ifndef _ARCH_ARM64_TLBBATCH_H
#define _ARCH_ARM64_TLBBATCH_H
#include <linux/cpumask.h>
struct arch_tlbflush_unmap_batch {
#ifdef CONFIG_ARM64_ERRATUM_4193714
/*
* Track CPUs that need SME DVMSync on completion of this batch.
* Otherwise, the arm64 HW can do tlb shootdown, so we don't need to
* record cpumask for sending IPI
* For arm64, HW can do TLB shootdown, so we don't need to record a
* cpumask for sending IPIs.
*/
cpumask_var_t cpumask;
#endif
};
#endif /* _ARCH_ARM64_TLBBATCH_H */

49
arch/arm64/include/asm/tlbflush.h
View File

@ -82,6 +82,8 @@ static inline unsigned long get_trans_granule(void)
#ifdef CONFIG_ARM64_ERRATUM_4193714
extern cpumask_t sme_active_cpus;
void sme_do_dvmsync(const struct cpumask *mask);
static inline void sme_dvmsync(struct mm_struct *mm)
@ -92,42 +94,12 @@ static inline void sme_dvmsync(struct mm_struct *mm)
sme_do_dvmsync(mm_cpumask(mm));
}
static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch,
struct mm_struct *mm)
static inline void sme_dvmsync_batch(void)
{
if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
return;
/*
* Order the mm_cpumask() read after the hardware DVMSync.
*/
dsb(ish);
if (cpumask_empty(mm_cpumask(mm)))
return;
/*
* Allocate the batch cpumask on first use. Fall back to an immediate
* IPI for this mm in case of failure.
*/
if (!cpumask_available(batch->cpumask) &&
!zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) {
sme_do_dvmsync(mm_cpumask(mm));
return;
}
cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm));
}
static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch)
{
if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
return;
if (!cpumask_available(batch->cpumask))
return;
sme_do_dvmsync(batch->cpumask);
cpumask_clear(batch->cpumask);
sme_do_dvmsync(&sme_active_cpus);
}
#else
@ -135,11 +107,7 @@ static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch)
static inline void sme_dvmsync(struct mm_struct *mm)
{
}
static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch,
struct mm_struct *mm)
{
}
static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch)
static inline void sme_dvmsync_batch(void)
{
}
@ -285,11 +253,11 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm)
sme_dvmsync(mm);
}
static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch)
static inline void __tlbi_sync_s1ish_batch(void)
{
dsb(ish);
__repeat_tlbi_sync(vale1is, 0);
sme_dvmsync_batch(batch);
sme_dvmsync_batch();
}
static inline void __tlbi_sync_s1ish_kernel(void)
@ -434,7 +402,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
*/
static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
__tlbi_sync_s1ish_batch(batch);
__tlbi_sync_s1ish_batch();
}
/*
@ -722,7 +690,6 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b
__flush_tlb_range(&vma, start, end, PAGE_SIZE, 3,
TLBF_NOWALKCACHE | TLBF_NOSYNC);
sme_dvmsync_add_pending(batch, mm);
}
static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval)

10
arch/arm64/kernel/fpsimd.c
View File

@ -1355,6 +1355,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs)
* SME/CME erratum handling.
*/
static cpumask_t sme_dvmsync_cpus;
cpumask_t sme_active_cpus;
/*
* These helpers are only called from non-preemptible contexts, so
@ -1368,13 +1369,15 @@ void sme_set_active(void)
return;
cpumask_set_cpu(cpu, mm_cpumask(current->mm));
cpumask_set_cpu(cpu, &sme_active_cpus);
/*
* A subsequent (post ERET) SME access may use a stale address
* translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for
* the completion of cpumask_set_cpu() above as it appears in program
* order before the SME access. The post-TLBI+DSB read of mm_cpumask()
* will lead to the IPI being issued.
* the completion of the cpumask_set_cpu() operations above as they
* appear in program order before the SME access. The post-TLBI+DSB
* read of mm_cpumask() or sme_active_cpus will lead to the IPI being
* issued.
*
* https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3
*/
@ -1392,6 +1395,7 @@ void sme_clear_active(void)
* completed on entering EL1.
*/
cpumask_clear_cpu(cpu, mm_cpumask(current->mm));
cpumask_clear_cpu(cpu, &sme_active_cpus);
}
static void sme_dvmsync_ipi(void *unused)

35
arch/arm64/kernel/process.c
View File

@ -341,41 +341,8 @@ void flush_thread(void)
flush_gcs();
}
#ifdef CONFIG_ARM64_ERRATUM_4193714
static void arch_dup_tlbbatch_mask(struct task_struct *dst)
{
/*
* Clear the inherited cpumask with memset() to cover both cases where
* cpumask_var_t is a pointer or an array. It will be allocated lazily
* in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y.
*/
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
memset(&dst->tlb_ubc.arch.cpumask, 0,
sizeof(dst->tlb_ubc.arch.cpumask));
}
static void arch_release_tlbbatch_mask(struct task_struct *tsk)
{
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
free_cpumask_var(tsk->tlb_ubc.arch.cpumask);
}
#else
static void arch_dup_tlbbatch_mask(struct task_struct *dst)
{
}
static void arch_release_tlbbatch_mask(struct task_struct *tsk)
{
}
#endif /* CONFIG_ARM64_ERRATUM_4193714 */
void arch_release_task_struct(struct task_struct *tsk)
{
arch_release_tlbbatch_mask(tsk);
fpsimd_release_task(tsk);
}
@ -391,8 +358,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
*dst = *src;
arch_dup_tlbbatch_mask(dst);
/*
* Drop stale reference to src's sve_state and convert dst to
* non-streaming FPSIMD mode.