diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 0fb3a2a62eb0..feb32949aeea 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -180,7 +180,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, CPUHP_AP_HRTIMERS_DYING, diff --git a/include/linux/sched.h b/include/linux/sched.h index 3188fc78ac9b..193a4a4dcc27 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -821,6 +821,17 @@ struct kmap_ctrl { #endif }; +#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPTION) +struct task_ipi_mask { + union { + cpumask_t *ipi_mask_ptr; + unsigned long ipi_mask_val; + }; +}; +#else +struct task_ipi_mask { }; +#endif + struct task_struct { #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -1358,6 +1369,7 @@ struct task_struct { struct list_head perf_event_list; struct perf_ctx_data __rcu *perf_ctx_data; #endif + struct task_ipi_mask __private ipi_mask; #ifdef CONFIG_DEBUG_PREEMPT unsigned long preempt_disable_ip; #endif diff --git a/include/linux/smp.h b/include/linux/smp.h index 6925d15ccaa7..2dfa7390717a 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -47,8 +47,7 @@ extern void __smp_call_single_queue(int cpu, struct llist_node *node); /* total number of cpus in this system (may exceed NR_CPUS) */ extern unsigned int total_cpus; -int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, - int wait); +int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, bool wait); void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void *info, bool wait, const struct cpumask *mask); @@ -239,6 +238,18 @@ static inline int get_boot_cpu_id(void) #endif /* !SMP */ +#if defined(CONFIG_PREEMPTION) && defined(CONFIG_SMP) +int smp_task_ipi_mask_alloc(struct task_struct *task); +void smp_task_ipi_mask_free(struct task_struct *task); +#else +static inline int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + return 0; +} + +static inline void smp_task_ipi_mask_free(struct task_struct *task) { } +#endif + /* * raw_smp_processor_id() - get the current (unstable) CPU id * diff --git a/kernel/fork.c b/kernel/fork.c index 175c73bbe2bf..1e68404bd773 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -537,6 +537,7 @@ void free_task(struct task_struct *tsk) #endif release_user_cpus_ptr(tsk); scs_release(tsk); + smp_task_ipi_mask_free(tsk); #ifndef CONFIG_THREAD_INFO_IN_TASK /* @@ -935,10 +936,14 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif account_kernel_stack(tsk, 1); - err = scs_prepare(tsk, node); + err = smp_task_ipi_mask_alloc(tsk); if (err) goto free_stack; + err = scs_prepare(tsk, node); + if (err) + goto free_ipi_mask; + #ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under @@ -1016,6 +1021,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif return tsk; +free_ipi_mask: + smp_task_ipi_mask_free(tsk); free_stack: exit_task_stack_account(tsk); free_thread_stack(tsk); diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 327c315f411c..2082f9b44370 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -348,6 +348,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra int ret = 0; struct scf_check *scfcp = NULL; struct scf_selector *scfsp = scf_sel_rand(trsp); + bool is_single = (scfsp->scfs_prim == SCF_PRIM_SINGLE || + scfsp->scfs_prim == SCF_PRIM_SINGLE_RPC); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc_obj(*scfcp, GFP_ATOMIC); @@ -364,8 +366,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); switch (scfsp->scfs_prim) { case SCF_PRIM_RESCHED: if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) { @@ -411,13 +411,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra if (!ret) { if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); + wait_for_completion(&scfcp->scfc_completion); if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); } else { scfp->n_single_rpc_ofl++; scf_add_to_free_list(scfcp); @@ -452,7 +449,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfcp->scfc_out = true; } if (scfcp && scfsp->scfs_wait) { - if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) && + if (WARN_ON_ONCE(((use_cpus_read_lock && num_online_cpus() > 1) || is_single) && !scfcp->scfc_out)) { pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim); atomic_inc(&n_mb_out_errs); // Leak rather than trash! @@ -463,8 +460,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); if (allocfail) schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete. else if (!(torture_random(trsp) & 0xfff)) diff --git a/kernel/smp.c b/kernel/smp.c index 52dffc86555c..b696bcc60c08 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -63,7 +64,14 @@ int smpcfd_prepare_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); return -ENOMEM; } - cfd->csd = alloc_percpu(call_single_data_t); + + /* + * Allocate the per-CPU CSD the first time a CPU comes up. It is + * not freed when the CPU is offlined, so csd_lock_wait() can access + * it even when the CPU was offlined after preemption was re-enabled. + */ + if (!cfd->csd) + cfd->csd = alloc_percpu(call_single_data_t); if (!cfd->csd) { free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); @@ -79,7 +87,6 @@ int smpcfd_dead_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); - free_percpu(cfd->csd); return 0; } @@ -182,16 +189,22 @@ static atomic_t csd_bug_count = ATOMIC_INIT(0); static void __csd_lock_record(call_single_data_t *csd) { if (!csd) { - smp_mb(); /* NULL cur_csd after unlock. */ - __this_cpu_write(cur_csd, NULL); + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): orders any preceding CSD + * callback/unlock before a remote reader observes NULL. + */ + smp_store_release(this_cpu_ptr(&cur_csd), NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); - smp_wmb(); /* func and info before csd. */ - __this_cpu_write(cur_csd, csd); - smp_mb(); /* Update cur_csd before function call. */ - /* Or before unlock, as the case may be. */ + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): publishes cur_csd_func and + * cur_csd_info before the non-NULL pointer becomes visible. + */ + smp_store_release(this_cpu_ptr(&cur_csd), csd); } static __always_inline void csd_lock_record(call_single_data_t *csd) @@ -272,7 +285,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in cpux = 0; else cpux = cpu; - cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ + /* + * Pairs with smp_store_release() of cur_csd in __csd_lock_record(): + * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info + * are the matching publication; a NULL value is ordered after any + * preceding CSD callback/unlock on the remote CPU. + */ + cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* How long since this CSD lock was stuck. */ ts_delta = ts2 - ts0; pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", @@ -323,6 +342,8 @@ static void __csd_lock_wait(call_single_data_t *csd) int bug_id = 0; u64 ts0, ts1; + guard(preempt)(); + ts1 = ts0 = ktime_get_mono_fast_ns(); for (;;) { if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages)) @@ -659,17 +680,9 @@ void flush_smp_call_function_queue(void) local_irq_restore(flags); } -/** - * smp_call_function_single - Run a function on a specific CPU - * @cpu: Specific target CPU for this function. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed on other CPUs. - * - * Returns: %0 on success, else a negative status code. - */ -int smp_call_function_single(int cpu, smp_call_func_t func, void *info, - int wait) +static int __smp_call_function_single(int cpu, smp_call_func_t func, + void *info, const struct cpumask *mask, + bool wait) { call_single_data_t *csd; call_single_data_t csd_stack = { @@ -686,6 +699,14 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, */ this_cpu = get_cpu(); + if (mask) { + /* Try for same CPU (cheapest) */ + if (!cpumask_test_cpu(this_cpu, mask)) + cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu)); + else + cpu = this_cpu; + } + /* * Can deadlock when called with interrupts disabled. * We allow cpu's that are not yet online though, as no one else can @@ -718,13 +739,32 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, err = generic_exec_single(cpu, csd); + /* + * @csd is stack-allocated when @wait is true. No concurrent access + * except from the IPI completion path, so we can re-enable preemption + * early to reduce latency. + */ + put_cpu(); + if (wait) csd_lock_wait(csd); - put_cpu(); - return err; } + +/** + * smp_call_function_single - Run a function on a specific CPU + * @cpu: Specific target CPU for this function. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed on other CPUs. + * + * Returns: %0 on success, else a negative status code. + */ +int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait) +{ + return __smp_call_function_single(cpu, func, info, NULL, wait); +} EXPORT_SYMBOL(smp_call_function_single); /** @@ -775,10 +815,10 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async); /** * smp_call_function_any - Run a function on any of the given cpus - * @mask: The mask of cpus it can run on. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed. + * @mask: The mask of cpus it can run on. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed. * * Selection preference: * 1) current cpu if in @mask @@ -789,20 +829,54 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async); int smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, void *info, int wait) { - unsigned int cpu; - int ret; - - /* Try for same CPU (cheapest) */ - cpu = get_cpu(); - if (!cpumask_test_cpu(cpu, mask)) - cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu)); - - ret = smp_call_function_single(cpu, func, info, wait); - put_cpu(); - return ret; + return __smp_call_function_single(-1, func, info, mask, wait); } EXPORT_SYMBOL_GPL(smp_call_function_any); +static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined); + +#ifdef CONFIG_PREEMPTION + +int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return 0; + + ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr = + kmalloc(cpumask_size(), GFP_KERNEL); + if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr) + return -ENOMEM; + + return 0; +} + +void smp_task_ipi_mask_free(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return; + + kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr); +} + +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + /* + * If cpumask_size() is smaller than or equal to the pointer + * size, it stashes the cpumask in the pointer itself to + * avoid extra memory allocations. + */ + if (static_branch_unlikely(&ipi_mask_inlined)) + return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val; + + return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr; +} +#else +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + return NULL; +} +#endif + /* * Flags to be used as scf_flags argument of smp_call_function_many_cond(). * @@ -817,13 +891,20 @@ static void smp_call_function_many_cond(const struct cpumask *mask, unsigned int scf_flags, smp_cond_func_t cond_func) { - int cpu, last_cpu, this_cpu = smp_processor_id(); - struct call_function_data *cfd; + struct cpumask *cpumask, *task_mask; bool wait = scf_flags & SCF_WAIT; - int nr_cpus = 0; + struct call_function_data *cfd; + int cpu, last_cpu, this_cpu; bool run_remote = false; + int nr_cpus = 0; - lockdep_assert_preemption_disabled(); + this_cpu = get_cpu(); + cfd = this_cpu_ptr(&cfd_data); + task_mask = smp_task_ipi_mask(current); + if (task_mask) + cpumask = task_mask; + else + cpumask = cfd->cpumask; /* * Can deadlock when called with interrupts disabled. @@ -845,16 +926,15 @@ static void smp_call_function_many_cond(const struct cpumask *mask, /* Check if we need remote execution, i.e., any CPU excluding this one. */ if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) { - cfd = this_cpu_ptr(&cfd_data); - cpumask_and(cfd->cpumask, mask, cpu_online_mask); - __cpumask_clear_cpu(this_cpu, cfd->cpumask); + cpumask_and(cpumask, mask, cpu_online_mask); + __cpumask_clear_cpu(this_cpu, cpumask); cpumask_clear(cfd->cpumask_ipi); - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); if (cond_func && !cond_func(cpu, info)) { - __cpumask_clear_cpu(cpu, cfd->cpumask); + __cpumask_clear_cpu(cpu, cpumask); continue; } @@ -904,8 +984,18 @@ static void smp_call_function_many_cond(const struct cpumask *mask, local_irq_restore(flags); } + /* + * The IPI work has been queued and dispatched. On PREEMPT kernels, + * tasks created through dup_task_struct() have task-local wait masks. + * The boot init_task can fall back to cfd->cpumask when the mask is + * not inlined, but other tasks still use task-local masks and cannot + * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule + * another task, so the per-CPU mask remains protected. + */ + put_cpu(); + if (run_remote && wait) { - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd; csd = per_cpu_ptr(cfd->csd, cpu); @@ -916,15 +1006,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask, /** * smp_call_function_many() - Run a function on a set of CPUs. - * @mask: The set of cpus to run on (only runs on online subset). - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @mask: The set of cpus to run on (only runs on online subset). + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. Preemption - * must be disabled when calling this function. + * hardware interrupt handler or from a bottom half handler. * * @func is not called on the local CPU even if @mask contains it. Consider * using on_each_cpu_cond_mask() instead if this is not desirable. @@ -938,10 +1027,10 @@ EXPORT_SYMBOL(smp_call_function_many); /** * smp_call_function() - Run a function on all other CPUs. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * If @wait is true, then returns once @func has returned; otherwise * it returns just before the target cpu calls @func. @@ -951,9 +1040,8 @@ EXPORT_SYMBOL(smp_call_function_many); */ void smp_call_function(smp_call_func_t func, void *info, int wait) { - preempt_disable(); - smp_call_function_many(cpu_online_mask, func, info, wait); - preempt_enable(); + smp_call_function_many_cond(cpu_online_mask, func, info, + wait ? SCF_WAIT : 0, NULL); } EXPORT_SYMBOL(smp_call_function); @@ -1019,6 +1107,9 @@ EXPORT_SYMBOL(nr_cpu_ids); void __init setup_nr_cpu_ids(void) { set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); + + if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long)) + static_branch_enable(&ipi_mask_inlined); } /* Called by boot processor to activate the rest. */ @@ -1055,12 +1146,14 @@ void __init smp_init(void) * @func: The function to run on all applicable CPUs. * This must be fast and non-blocking. * @info: An arbitrary pointer to pass to both functions. - * @wait: If true, wait (atomically) until function has - * completed on other CPUs. + * @wait: If true, wait until function has completed on other CPUs. * @mask: The set of cpus to run on (only runs on online subset). * - * Preemption is disabled to protect against CPUs going offline but not online. - * CPUs going online during the call will not be seen or sent an IPI. + * Target CPU selection and work queueing are done with preemption + * disabled. This protects against CPUs going offline, but not against + * CPUs coming online concurrently; newly online CPUs are not guaranteed + * to be seen or sent an IPI. If @wait is true, the final wait for remote + * completion happens after that preemption-disabled section. * * You must not call this function with disabled interrupts or * from a hardware interrupt handler or from a bottom half handler. @@ -1073,9 +1166,7 @@ void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, if (wait) scf_flags |= SCF_WAIT; - preempt_disable(); smp_call_function_many_cond(mask, func, info, scf_flags, cond_func); - preempt_enable(); } EXPORT_SYMBOL(on_each_cpu_cond_mask); diff --git a/kernel/up.c b/kernel/up.c index df50828cc2f0..6d4ac9502e8b 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -9,8 +9,7 @@ #include #include -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int wait) +int smp_call_function_single(int cpu, void (*func)(void *info), void *info, bool wait) { unsigned long flags;