From ec9f57e6fefbc0497b9f047ebe60ef9adaae7480 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 29 Jun 2026 03:57:45 -0700 Subject: [PATCH 01/11] smp: Use release stores for csd_lock_record() state __csd_lock_record() publishes per-CPU diagnostic state (cur_csd, cur_csd_func, cur_csd_info) that is consumed from a remote CPU by csd_lock_wait_toolong() via smp_load_acquire(&cur_csd). To order the matching cur_csd_func/cur_csd_info stores before the cur_csd publication, the producer issues smp_wmb() before writing cur_csd; to order the publication before the subsequent callback execution or CSD unlock, it issues smp_mb() after the write. The clear path mirrors this with smp_mb() before storing NULL into cur_csd so the preceding callback/unlock is observed first. The smp_mb() pair is heavier than what the consumer actually requires (on x86 each emits a locked full barrier). The consumer only needs to observe the matching cur_csd_func/cur_csd_info when it sees a non-NULL cur_csd, and to observe the preceding callback/unlock when it sees NULL -- both of which a release/acquire pair provides. The extra two-way ordering enforced by smp_mb() -- that cur_csd publication be observed before callback execution or unlock becomes visible -- would only matter if cur_csd were an exact live-state marker. csd_lock_wait_toolong() does not treat it that way: it snapshots cur_csd via smp_load_acquire() and then prints / dumps / re-IPIs without an RCU-style stall-ended recheck, so the diagnostic already tolerates the remote CPU completing its work between snapshot and report. cur_csd is best-effort context, not a precise stall boundary. Replace the smp_wmb() + plain store + smp_mb() in the publish path, and the smp_mb() + plain store in the clear path, with smp_store_release(). This pairs with the smp_load_acquire() in csd_lock_wait_toolong(): preceding cur_csd_func/cur_csd_info stores become visible before a remote reader observes the non-NULL publication, and any preceding callback/unlock becomes visible before a reader observes the NULL clear. Signed-off-by: Usama Arif Signed-off-by: Thomas Gleixner Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/20260629105745.1696683-1-usama.arif@linux.dev --- kernel/smp.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index a0bb56bd8dda..8a847a34f132 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -182,16 +182,22 @@ static atomic_t csd_bug_count = ATOMIC_INIT(0); static void __csd_lock_record(call_single_data_t *csd) { if (!csd) { - smp_mb(); /* NULL cur_csd after unlock. */ - __this_cpu_write(cur_csd, NULL); + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): orders any preceding CSD + * callback/unlock before a remote reader observes NULL. + */ + smp_store_release(this_cpu_ptr(&cur_csd), NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); - smp_wmb(); /* func and info before csd. */ - __this_cpu_write(cur_csd, csd); - smp_mb(); /* Update cur_csd before function call. */ - /* Or before unlock, as the case may be. */ + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): publishes cur_csd_func and + * cur_csd_info before the non-NULL pointer becomes visible. + */ + smp_store_release(this_cpu_ptr(&cur_csd), csd); } static __always_inline void csd_lock_record(call_single_data_t *csd) @@ -272,7 +278,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in cpux = 0; else cpux = cpu; - cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ + /* + * Pairs with smp_store_release() of cur_csd in __csd_lock_record(): + * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info + * are the matching publication; a NULL value is ordered after any + * preceding CSD callback/unlock on the remote CPU. + */ + cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* How long since this CSD lock was stuck. */ ts_delta = ts2 - ts0; pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", From aeec9d9ae6143db48a01d8acc64c940b4e65d762 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Sat, 11 Jul 2026 21:51:54 +0800 Subject: [PATCH 02/11] cpu/hotplug: Remove CPUHP_AP_ARM_CORESIGHT_CTI_STARTING Thestate CPUHP_AP_ARM_CORESIGHT_CTI_STARTING has become unused since commit 59213b4be5c1 ("coresight: cti: Remove CPU power management code"). Remove it as well. Signed-off-by: Zenghui Yu Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260711135154.94276-1-zenghui.yu@linux.dev --- include/linux/cpuhotplug.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 0fb3a2a62eb0..feb32949aeea 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -180,7 +180,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, CPUHP_AP_HRTIMERS_DYING, From b46883305f26188bf0555d06e77f43ab01d5ec95 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:20 +0800 Subject: [PATCH 03/11] smp: Disable preemption explicitly in __csd_lock_wait() The CSD lock wait debugging code in __csd_lock_wait() must run with preemption disabled. The smp function call mechanisms which invoke it currently keep preemption disabled across the wait, so the debugging code inherits that guarantee from its callers. Keeping preemption disabled across the whole smp function call operation can induce large scheduling latencies. Shortening the caller-side preemption-disabled region would invoke __csd_lock_wait() with preemption enabled. Prepare for that by disabling preemption explicitly around the CSD lock wait debugging code in __csd_lock_wait(). Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Steven Rostedt (Google) Reviewed-by: Sebastian Andrzej Siewior Acked-by: Muchun Song Link: https://patch.msgid.link/20260709122933.4021501-2-zhouchuyi@bytedance.com --- kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/smp.c b/kernel/smp.c index 8a847a34f132..f5a4d63f3231 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -335,6 +335,8 @@ static void __csd_lock_wait(call_single_data_t *csd) int bug_id = 0; u64 ts0, ts1; + guard(preempt)(); + ts1 = ts0 = ktime_get_mono_fast_ns(); for (;;) { if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages)) From cdf0ba15c0898a04f55ca396d119b97bbcd801bc Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:21 +0800 Subject: [PATCH 04/11] smp: Enable preemption early in smp_call_function_single() smp_call_function_single() disables preemption while it validates the target CPU, prepares the call single data, queues the callback and sends the IPI. For the !wait case, preemption protects the per-CPU csd_data from concurrent modification by another task on the same CPU. For the wait case, the CSD is stack allocated and no other task can reuse it. CPU pinning is still required until the callback has been queued and the IPI has been sent, to ensure that the target CPU cannot be offlined after the online check but before dispatch. After generic_exec_single() has queued the callback, the synchronous csd_lock_wait() invocation at the end of the execution does not require the caller to remain pinned to the current CPU. Enable preemption before csd_lock_wait() to shorten the preemption-disabled section. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Muchun Song Reviewed-by: Steven Rostedt (Google) Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-3-zhouchuyi@bytedance.com --- kernel/smp.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index f5a4d63f3231..31cdb06a734a 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -712,11 +712,16 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, err = generic_exec_single(cpu, csd); + /* + * @csd is stack-allocated when @wait is true. No concurrent access + * except from the IPI completion path, so we can re-enable preemption + * early to reduce latency. + */ + put_cpu(); + if (wait) csd_lock_wait(csd); - put_cpu(); - return err; } EXPORT_SYMBOL(smp_call_function_single); From 0485235b6e0dcb8c70e5c947faa2b5031febd736 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:22 +0800 Subject: [PATCH 05/11] smp: Refactor remote CPU selection in smp_call_function_any() smp_call_function_any() disables preemption across the entire operation: selecting a target CPU, enqueueing the IPI, and synchronously waiting for the remote CPU. smp_call_function_single() already re-enables preemption before the synchronous csd_lock_wait(), so callers of smp_call_function_any() should benefit from the same shorter preemption-disabled section. Simply removing get_cpu() and put_cpu() from smp_call_function_any() would leave the preemption disablement entirely to smp_call_function_single(). That opens a preemption window between selecting the remote CPU, for example via sched_numa_find_nth_cpu(), and dispatching the IPI in smp_call_function_single(). If the selected CPU is fully offlined in that window, smp_call_function_single() fails its cpu_online() check and returns -ENXIO to the caller, violating the guarantee that smp_call_function_any() executes on any online CPU in the mask. Move the remote CPU selection into a common __smp_call_function_single() helper. Keep the target CPU selection and IPI dispatch within the same preemption-disabled region, while still allowing the wait path to use the shorter preemption-disabled section provided by smp_call_function_single(). Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-4-zhouchuyi@bytedance.com --- include/linux/smp.h | 3 +-- kernel/smp.c | 56 ++++++++++++++++++++++++--------------------- kernel/up.c | 3 +-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/include/linux/smp.h b/include/linux/smp.h index 6925d15ccaa7..11e36c7bc4d6 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -47,8 +47,7 @@ extern void __smp_call_single_queue(int cpu, struct llist_node *node); /* total number of cpus in this system (may exceed NR_CPUS) */ extern unsigned int total_cpus; -int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, - int wait); +int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, bool wait); void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void *info, bool wait, const struct cpumask *mask); diff --git a/kernel/smp.c b/kernel/smp.c index 31cdb06a734a..5c0502982cef 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -653,17 +653,9 @@ void flush_smp_call_function_queue(void) local_irq_restore(flags); } -/** - * smp_call_function_single - Run a function on a specific CPU - * @cpu: Specific target CPU for this function. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed on other CPUs. - * - * Returns: %0 on success, else a negative status code. - */ -int smp_call_function_single(int cpu, smp_call_func_t func, void *info, - int wait) +static int __smp_call_function_single(int cpu, smp_call_func_t func, + void *info, const struct cpumask *mask, + bool wait) { call_single_data_t *csd; call_single_data_t csd_stack = { @@ -680,6 +672,14 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, */ this_cpu = get_cpu(); + if (mask) { + /* Try for same CPU (cheapest) */ + if (!cpumask_test_cpu(this_cpu, mask)) + cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu)); + else + cpu = this_cpu; + } + /* * Can deadlock when called with interrupts disabled. * We allow cpu's that are not yet online though, as no one else can @@ -724,6 +724,20 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, return err; } + +/** + * smp_call_function_single - Run a function on a specific CPU + * @cpu: Specific target CPU for this function. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed on other CPUs. + * + * Returns: %0 on success, else a negative status code. + */ +int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait) +{ + return __smp_call_function_single(cpu, func, info, NULL, wait); +} EXPORT_SYMBOL(smp_call_function_single); /** @@ -774,10 +788,10 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async); /** * smp_call_function_any - Run a function on any of the given cpus - * @mask: The mask of cpus it can run on. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed. + * @mask: The mask of cpus it can run on. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed. * * Selection preference: * 1) current cpu if in @mask @@ -788,17 +802,7 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async); int smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, void *info, int wait) { - unsigned int cpu; - int ret; - - /* Try for same CPU (cheapest) */ - cpu = get_cpu(); - if (!cpumask_test_cpu(cpu, mask)) - cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu)); - - ret = smp_call_function_single(cpu, func, info, wait); - put_cpu(); - return ret; + return __smp_call_function_single(-1, func, info, mask, wait); } EXPORT_SYMBOL_GPL(smp_call_function_any); diff --git a/kernel/up.c b/kernel/up.c index df50828cc2f0..6d4ac9502e8b 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -9,8 +9,7 @@ #include #include -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int wait) +int smp_call_function_single(int cpu, void (*func)(void *info), void *info, bool wait) { unsigned long flags; From 9a560af15fd31b3b93235b1584a2509445a17963 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:23 +0800 Subject: [PATCH 06/11] smp: Use task-local IPI cpumask in smp_call_function_many_cond() smp_call_function_many_cond() uses the per-CPU cfd->cpumask as the list of remote CPUs to wait for. That is safe while the caller remains pinned to the current CPU for the whole operation, because another task cannot run on the same CPU and reuse the per-CPU mask. The synchronous wait is the long-latency part of the operation. To make that wait preemptible, the mask iterated by csd_lock_wait() must remain stable even if the task is preempted or migrates. If the wait used the per-CPU cfd->cpumask after dropping CPU pinning, another task scheduled on the original CPU could enter smp_call_function_many_cond() and overwrite the mask while the first task is still iterating it. Give each task private IPI cpumask storage and use it as the wait mask in smp_call_function_many_cond(). Other cpumask storage choices do not fit this use case: - Per-CPU storage is the state that becomes unsafe once the wait is made preemptible. After the caller drops CPU pinning, another task scheduled on the original CPU can enter smp_call_function_many_cond() and reuse the same per-CPU mask. - Stack storage is not suitable for large NR_CPUS or CONFIG_CPUMASK_OFFSTACK=y configurations. The wait mask needs to scale with cpumask_size(), and putting that storage on the stack is not acceptable on large systems. - Allocating the mask inside smp_call_function_many_cond() would put an allocation and a failure path in the generic IPI path. A sleeping allocation is not suitable because callers have historically only provided a preempt-disabled context, not a sleepable one. GFP_ATOMIC would avoid sleeping, but a failure fallback would make the latency improvement opportunistic instead of guaranteed. The users are not limited to a small, pre-identifiable class of tasks. On x86, ordinary tasks can reach this path through TLB flushes during exit, unmap and reclaim, so allocating the mask only for a known subset of tasks is not straightforward. The memory cost is explicit: one word is added to task_struct. When cpumask_size() fits in that word, the mask is stored inline and no separate allocation is needed. Larger systems allocate cpumask_size() per task; on x86-64 NR_CPUS=8192 this is 1 KiB per task. For context, x86 already carries several KiB of per-task architecture and FPU state, depending on the enabled features and configuration. That does not make the extra cpumask free, but it puts the large-NR_CPUS case in perspective. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-5-zhouchuyi@bytedance.com --- include/linux/sched.h | 12 ++++++++ include/linux/smp.h | 12 ++++++++ kernel/fork.c | 9 +++++- kernel/smp.c | 71 ++++++++++++++++++++++++++++++++++++++----- 4 files changed, 95 insertions(+), 9 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 373bcc0598d1..5738c54eb85a 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -823,6 +823,17 @@ struct kmap_ctrl { #endif }; +#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPTION) +struct task_ipi_mask { + union { + cpumask_t *ipi_mask_ptr; + unsigned long ipi_mask_val; + }; +}; +#else +struct task_ipi_mask { }; +#endif + struct task_struct { #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -1359,6 +1370,7 @@ struct task_struct { struct list_head perf_event_list; struct perf_ctx_data __rcu *perf_ctx_data; #endif + struct task_ipi_mask __private ipi_mask; #ifdef CONFIG_DEBUG_PREEMPT unsigned long preempt_disable_ip; #endif diff --git a/include/linux/smp.h b/include/linux/smp.h index 11e36c7bc4d6..2dfa7390717a 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -238,6 +238,18 @@ static inline int get_boot_cpu_id(void) #endif /* !SMP */ +#if defined(CONFIG_PREEMPTION) && defined(CONFIG_SMP) +int smp_task_ipi_mask_alloc(struct task_struct *task); +void smp_task_ipi_mask_free(struct task_struct *task); +#else +static inline int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + return 0; +} + +static inline void smp_task_ipi_mask_free(struct task_struct *task) { } +#endif + /* * raw_smp_processor_id() - get the current (unstable) CPU id * diff --git a/kernel/fork.c b/kernel/fork.c index 13e38e89a1f3..ac3fc49d90ab 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -537,6 +537,7 @@ void free_task(struct task_struct *tsk) #endif release_user_cpus_ptr(tsk); scs_release(tsk); + smp_task_ipi_mask_free(tsk); #ifndef CONFIG_THREAD_INFO_IN_TASK /* @@ -935,10 +936,14 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif account_kernel_stack(tsk, 1); - err = scs_prepare(tsk, node); + err = smp_task_ipi_mask_alloc(tsk); if (err) goto free_stack; + err = scs_prepare(tsk, node); + if (err) + goto free_ipi_mask; + #ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under @@ -1011,6 +1016,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif return tsk; +free_ipi_mask: + smp_task_ipi_mask_free(tsk); free_stack: exit_task_stack_account(tsk); free_thread_stack(tsk); diff --git a/kernel/smp.c b/kernel/smp.c index 5c0502982cef..19fdee6bf3eb 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -806,6 +807,50 @@ int smp_call_function_any(const struct cpumask *mask, } EXPORT_SYMBOL_GPL(smp_call_function_any); +static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined); + +#ifdef CONFIG_PREEMPTION + +int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return 0; + + ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr = + kmalloc(cpumask_size(), GFP_KERNEL); + if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr) + return -ENOMEM; + + return 0; +} + +void smp_task_ipi_mask_free(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return; + + kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr); +} + +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + /* + * If cpumask_size() is smaller than or equal to the pointer + * size, it stashes the cpumask in the pointer itself to + * avoid extra memory allocations. + */ + if (static_branch_unlikely(&ipi_mask_inlined)) + return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val; + + return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr; +} +#else +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + return NULL; +} +#endif + /* * Flags to be used as scf_flags argument of smp_call_function_many_cond(). * @@ -821,13 +866,21 @@ static void smp_call_function_many_cond(const struct cpumask *mask, smp_cond_func_t cond_func) { int cpu, last_cpu, this_cpu = smp_processor_id(); - struct call_function_data *cfd; + struct cpumask *cpumask, *task_mask; bool wait = scf_flags & SCF_WAIT; - int nr_cpus = 0; + struct call_function_data *cfd; bool run_remote = false; + int nr_cpus = 0; lockdep_assert_preemption_disabled(); + cfd = this_cpu_ptr(&cfd_data); + task_mask = smp_task_ipi_mask(current); + if (task_mask) + cpumask = task_mask; + else + cpumask = cfd->cpumask; + /* * Can deadlock when called with interrupts disabled. * We allow cpu's that are not yet online though, as no one else can @@ -848,16 +901,15 @@ static void smp_call_function_many_cond(const struct cpumask *mask, /* Check if we need remote execution, i.e., any CPU excluding this one. */ if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) { - cfd = this_cpu_ptr(&cfd_data); - cpumask_and(cfd->cpumask, mask, cpu_online_mask); - __cpumask_clear_cpu(this_cpu, cfd->cpumask); + cpumask_and(cpumask, mask, cpu_online_mask); + __cpumask_clear_cpu(this_cpu, cpumask); cpumask_clear(cfd->cpumask_ipi); - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); if (cond_func && !cond_func(cpu, info)) { - __cpumask_clear_cpu(cpu, cfd->cpumask); + __cpumask_clear_cpu(cpu, cpumask); continue; } @@ -908,7 +960,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask, } if (run_remote && wait) { - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd; csd = per_cpu_ptr(cfd->csd, cpu); @@ -1022,6 +1074,9 @@ EXPORT_SYMBOL(nr_cpu_ids); void __init setup_nr_cpu_ids(void) { set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); + + if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long)) + static_branch_enable(&ipi_mask_inlined); } /* Called by boot processor to activate the rest. */ From 9f483e5b2f1263f7fad20bcc5b09d01d2da57f1f Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:24 +0800 Subject: [PATCH 07/11] smp: Alloc percpu csd data in smpcfd_prepare_cpu() only once smp_call_function_many_cond() uses per-CPU CSD objects when queueing callbacks to remote CPUs, and the wait path later dereferences those CSDs from csd_lock_wait(). Making the wait path preemptible allows the initiating task to be preempted or migrated before it waits for completion. A target CPU can be offlined in that window. If smpcfd_dead_cpu() frees the target CPU's per-CPU CSD storage, csd_lock_wait() can later dereference freed memory. One way to protect the CSD storage is to free it via RCU or after a synchronization step in the CPU offline path, but that would add unnecessary complexity and can delay CPU shutdown. Allocate the per-CPU CSD storage the first time a CPU comes up and keep it allocated when the CPU is offlined. This allows csd_lock_wait() to access the CSD even when the target CPU is offlined after preemption is re-enabled and before the wait is invoked. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Acked-by: Muchun Song Link: https://patch.msgid.link/20260709122933.4021501-6-zhouchuyi@bytedance.com --- kernel/smp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index 19fdee6bf3eb..e6c1d839c187 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -64,7 +64,14 @@ int smpcfd_prepare_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); return -ENOMEM; } - cfd->csd = alloc_percpu(call_single_data_t); + + /* + * Allocate the per-CPU CSD the first time a CPU comes up. It is + * not freed when the CPU is offlined, so csd_lock_wait() can access + * it even when the CPU was offlined after preemption was re-enabled. + */ + if (!cfd->csd) + cfd->csd = alloc_percpu(call_single_data_t); if (!cfd->csd) { free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); @@ -80,7 +87,6 @@ int smpcfd_dead_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); - free_percpu(cfd->csd); return 0; } From 8df8a6028309a549ba1b83cf4f01e8e0f7f23f3f Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:25 +0800 Subject: [PATCH 08/11] smp: Enable preemption early in smp_call_function_many_cond() smp_call_function_many_cond() still has to keep the caller pinned to the current CPU while the remote IPI request is built and dispatched. This protects the queueing state and CPU-hotplug boundary that are required before the synchronous wait starts: - It protects the current CPU's per-CPU scratch cpumask, cfd->cpumask_ipi. Another task running on the same CPU could otherwise enter smp_call_function_many_cond() and reuse that scratch cpumask before the current caller has finished building and sending the IPI request. - It provides the CPU-hotplug exclusion required by the CSD queueing side. New CSDs must not be queued after smpcfd_dying_cpu() has flushed the outgoing CPU's callback queue. Keeping preemption disabled until all required CSDs have been queued and the corresponding IPIs have been sent prevents CPU offline from crossing that boundary in the middle of the queueing operation. The CSD acquisition side also relies on that caller-side CPU pinning. csd_lock() waits for CSD_FLAG_LOCK to clear and then marks the CSD busy with a regular store, so another task on the same CPU must not be allowed to acquire and reinitialize the same per-CPU CSD concurrently. After the callbacks have been queued and the IPIs have been sent, the caller only performs the final csd_lock_wait() completion wait. If it is preempted there, another task running on the original CPU may enter smp_call_function_many_cond(), but any attempt to reuse the same per-CPU CSD will block in csd_lock() until the previous callback clears CSD_FLAG_LOCK. The final csd_lock_wait() does not acquire or reinitialize the CSD, so it does not need the same caller-side preemption-disabled protection. The wait mask is task-local, so it cannot be overwritten by another task on the original CPU. The per-CPU CSD storage also remains allocated across CPU offline, so csd_lock_wait() can safely dereference it even if the target CPU is offlined after the caller is unpinned. With those requirements satisfied, enable preemption before the synchronous csd_lock_wait() loop. This makes the potentially long wait preemptible and migratable while keeping the CPU-pinned section around the remote CPU selection and IPI dispatch. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-7-zhouchuyi@bytedance.com --- kernel/smp.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index e6c1d839c187..f6f8529a8003 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -871,15 +871,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask, unsigned int scf_flags, smp_cond_func_t cond_func) { - int cpu, last_cpu, this_cpu = smp_processor_id(); struct cpumask *cpumask, *task_mask; bool wait = scf_flags & SCF_WAIT; struct call_function_data *cfd; + int cpu, last_cpu, this_cpu; bool run_remote = false; int nr_cpus = 0; - lockdep_assert_preemption_disabled(); - + this_cpu = get_cpu(); cfd = this_cpu_ptr(&cfd_data); task_mask = smp_task_ipi_mask(current); if (task_mask) @@ -965,6 +964,16 @@ static void smp_call_function_many_cond(const struct cpumask *mask, local_irq_restore(flags); } + /* + * The IPI work has been queued and dispatched. On PREEMPT kernels, + * tasks created through dup_task_struct() have task-local wait masks. + * The boot init_task can fall back to cfd->cpumask when the mask is + * not inlined, but other tasks still use task-local masks and cannot + * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule + * another task, so the per-CPU mask remains protected. + */ + put_cpu(); + if (run_remote && wait) { for_each_cpu(cpu, cpumask) { call_single_data_t *csd; @@ -977,15 +986,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask, /** * smp_call_function_many() - Run a function on a set of CPUs. - * @mask: The set of cpus to run on (only runs on online subset). - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @mask: The set of cpus to run on (only runs on online subset). + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. Preemption - * must be disabled when calling this function. + * hardware interrupt handler or from a bottom half handler. * * @func is not called on the local CPU even if @mask contains it. Consider * using on_each_cpu_cond_mask() instead if this is not desirable. From 66344732b058e75f7ef25dcd0fbb483f9c000633 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:26 +0800 Subject: [PATCH 09/11] smp: Remove preempt_disable() from smp_call_function() smp_call_function_many_cond() handles the preemption and CPU pinning requirements internally. smp_call_function() only forwards the request to that helper for cpu_online_mask and does not access CPU-local state on its own. Remove the outer preempt_disable() and preempt_enable() pair from smp_call_function(). Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Muchun Song Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-8-zhouchuyi@bytedance.com --- kernel/smp.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index f6f8529a8003..6c4c93e6a5b5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -1007,10 +1007,10 @@ EXPORT_SYMBOL(smp_call_function_many); /** * smp_call_function() - Run a function on all other CPUs. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * If @wait is true, then returns once @func has returned; otherwise * it returns just before the target cpu calls @func. @@ -1020,9 +1020,8 @@ EXPORT_SYMBOL(smp_call_function_many); */ void smp_call_function(smp_call_func_t func, void *info, int wait) { - preempt_disable(); - smp_call_function_many(cpu_online_mask, func, info, wait); - preempt_enable(); + smp_call_function_many_cond(cpu_online_mask, func, info, + wait ? SCF_WAIT : 0, NULL); } EXPORT_SYMBOL(smp_call_function); From 947c397f5991b8ba9b5de2a5d3fb109ce663e75c Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:27 +0800 Subject: [PATCH 10/11] smp: Remove preempt_disable() from on_each_cpu_cond_mask() smp_call_function_many_cond() handles the preemption and CPU pinning requirements internally. on_each_cpu_cond_mask() only builds the call flags and forwards the request to that helper. Remove the outer preempt_disable() and preempt_enable() pair from on_each_cpu_cond_mask(). Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Muchun Song Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-9-zhouchuyi@bytedance.com --- kernel/smp.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index 6c4c93e6a5b5..6f8f4c9d8fd8 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -1126,12 +1126,14 @@ void __init smp_init(void) * @func: The function to run on all applicable CPUs. * This must be fast and non-blocking. * @info: An arbitrary pointer to pass to both functions. - * @wait: If true, wait (atomically) until function has - * completed on other CPUs. + * @wait: If true, wait until function has completed on other CPUs. * @mask: The set of cpus to run on (only runs on online subset). * - * Preemption is disabled to protect against CPUs going offline but not online. - * CPUs going online during the call will not be seen or sent an IPI. + * Target CPU selection and work queueing are done with preemption + * disabled. This protects against CPUs going offline, but not against + * CPUs coming online concurrently; newly online CPUs are not guaranteed + * to be seen or sent an IPI. If @wait is true, the final wait for remote + * completion happens after that preemption-disabled section. * * You must not call this function with disabled interrupts or * from a hardware interrupt handler or from a bottom half handler. @@ -1144,9 +1146,7 @@ void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, if (wait) scf_flags |= SCF_WAIT; - preempt_disable(); smp_call_function_many_cond(mask, func, info, scf_flags, cond_func); - preempt_enable(); } EXPORT_SYMBOL(on_each_cpu_cond_mask); From 99b49e02f9488335156c896c24aab0785623eb67 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:28 +0800 Subject: [PATCH 11/11] scftorture: Remove preempt_disable() in scftorture_invoke_one() The smp_call*() functions handle their required preemption and CPU pinning internally. The explicit preempt_disable() in scftorture_invoke_one() is therefore no longer required for correctness. Keeping the outer preempt_disable() would also prevent scftorture from exercising the narrowed internal preemption-disabled regions during IPI dispatch. Removing the outer preemption protection can expose a CPU hotplug race in the test validation when use_cpus_read_lock is false. For multicast operations, SCF_PRIM_MANY or SCF_PRIM_ALL, if only one CPU is online, smp_call_function_many() correctly skips sending IPIs and leaves scfc_out false. Without preemption disabled, a CPU hotplug thread can preempt the test thread, bring a second CPU online and increment num_online_cpus(). When the test thread resumes, the validation check can observe num_online_cpus() > 1 and falsely trigger the memory-ordering warning, leaking the scfcp structure. Remove the preempt_disable() and preempt_enable() pairs around the smp_call*() invocations in scftorture_invoke_one(). Restrict the num_online_cpus() > 1 validation to the use_cpus_read_lock=true case, where the CPU count is stable during the evaluation. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-10-zhouchuyi@bytedance.com --- kernel/scftorture.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 327c315f411c..2082f9b44370 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -348,6 +348,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra int ret = 0; struct scf_check *scfcp = NULL; struct scf_selector *scfsp = scf_sel_rand(trsp); + bool is_single = (scfsp->scfs_prim == SCF_PRIM_SINGLE || + scfsp->scfs_prim == SCF_PRIM_SINGLE_RPC); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc_obj(*scfcp, GFP_ATOMIC); @@ -364,8 +366,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); switch (scfsp->scfs_prim) { case SCF_PRIM_RESCHED: if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) { @@ -411,13 +411,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra if (!ret) { if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); + wait_for_completion(&scfcp->scfc_completion); if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); } else { scfp->n_single_rpc_ofl++; scf_add_to_free_list(scfcp); @@ -452,7 +449,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfcp->scfc_out = true; } if (scfcp && scfsp->scfs_wait) { - if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) && + if (WARN_ON_ONCE(((use_cpus_read_lock && num_online_cpus() > 1) || is_single) && !scfcp->scfc_out)) { pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim); atomic_inc(&n_mb_out_errs); // Leak rather than trash! @@ -463,8 +460,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); if (allocfail) schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete. else if (!(torture_random(trsp) & 0xfff))