From 8386ed68ae041b4d0919d851ec5300e41c1c2195 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 19 Aug 2026 07:37:35 -0700 Subject: [PATCH 01/16] workqueue: move the unbound-only attrs check to apply_workqueue_attrs() apply_workqueue_attrs_locked() refuses any workqueue without WQ_UNBOUND. That is the right rule for changing a workqueue's attributes, but it also blocks the internal callers, and building a percpu workqueue's pwq table through this path needs them. Move the check into apply_workqueue_attrs(), the exported entry point, which is the one that changes attributes on a live workqueue. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b8bec1689b7a..acd985ae2ac6 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5577,10 +5577,6 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, { struct apply_wqattrs_ctx *ctx; - /* only unbound workqueues can change attributes */ - if (WARN_ON(!(wq->flags & WQ_UNBOUND))) - return -EINVAL; - ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -5612,6 +5608,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, { int ret; + /* only unbound workqueues can change attributes */ + if (WARN_ON(!(wq->flags & WQ_UNBOUND))) + return -EINVAL; + mutex_lock(&wq_pool_mutex); ret = apply_workqueue_attrs_locked(wq, attrs); mutex_unlock(&wq_pool_mutex); From e92329ef60a8961a71028b7b9ca356c0b07bcc26 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 19 Aug 2026 07:37:36 -0700 Subject: [PATCH 02/16] workqueue: resolve the backing pool in alloc_pwq() Unify the pool allocation in alloc_pwq(), instead of only getting the unbound pool, and add a new parameter for the CPU. Create a helper for unbound allocations (alloc_unbound_pwq()). Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index acd985ae2ac6..f4d8cf0b6d51 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5385,22 +5385,34 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) return &per_cpu_ptr(pools, cpu)[highpri]; } -/* obtain a pool matching @attr and create a pwq associating the pool and @wq */ +/* + * Obtain the pool backing @wq on @cpu and create a pwq associating the two. + * A WQ_PERCPU workqueue is backed by the static per-cpu pool of @cpu, + * everything else by a pool matching @attrs. @cpu < 0 is always unbound. + */ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, - const struct workqueue_attrs *attrs) + const struct workqueue_attrs *attrs, + int cpu) { struct worker_pool *pool; struct pool_workqueue *pwq; lockdep_assert_held(&wq_pool_mutex); - pool = get_unbound_pool(attrs); - if (!pool) - return NULL; + WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0); + + if (cpu >= 0 && (wq->flags & WQ_PERCPU)) { + pool = get_percpu_pool(wq, cpu); + } else { + pool = get_unbound_pool(attrs); + if (!pool) + return NULL; + } pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); if (!pwq) { - put_unbound_pool(pool); + if (!is_percpu_pool(pool)) + put_unbound_pool(pool); return NULL; } @@ -5408,6 +5420,13 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, return pwq; } +/* create a pwq backed by an unbound pool matching @attrs */ +static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, + const struct workqueue_attrs *attrs) +{ + return alloc_pwq(wq, attrs, -1); +} + /** * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod * @attrs: the wq_attrs of the default pwq of the target workqueue @@ -5509,7 +5528,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_pwq(wq, new_attrs); + ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); if (!ctx->dfl_pwq) goto out_free; @@ -5519,7 +5538,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5664,7 +5683,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) return; /* create a new pwq */ - pwq = alloc_pwq(wq, target_attrs); + pwq = alloc_unbound_pwq(wq, target_attrs); if (!pwq) { pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", wq->name); From 2581b2b6d20abec5c9072b3cbaa9bdeaba131d7b Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 19 Aug 2026 07:37:37 -0700 Subject: [PATCH 03/16] workqueue: make the default pwq optional apply_wqattrs_prepare() always creates a dfl_pwq, the fallback a CPU uses when it has no pwq of its own. This is a CPU unbound field, so, we don't need it for the per cpu affinity version of workqueue. A percpu workqueue has no use for one. Skip the allocation for a WQ_PERCPU workqueue and skip installing what was not allocated. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f4d8cf0b6d51..10c40fe23e97 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5522,15 +5522,18 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, /* * If something goes wrong during CPU up/down, we'll fall back to - * the default pwq covering whole @attrs->cpumask. Always create - * it even if we don't use it immediately. + * the default pwq covering whole @attrs->cpumask. Create it even + * if we don't use it immediately. A percpu workqueue has a pwq on + * every possible CPU and never falls back, so it has no default. */ copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); - if (!ctx->dfl_pwq) - goto out_free; + if (!(wq->flags & WQ_PERCPU)) { + ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); + if (!ctx->dfl_pwq) + goto out_free; + } for_each_possible_cpu(cpu) { if (new_attrs->ordered) { @@ -5582,7 +5585,8 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) for_each_possible_cpu(cpu) ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, ctx->pwq_tbl[cpu]); - ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); + if (ctx->dfl_pwq) + ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); /* update node_nr_active->max, which only unbound workqueues have */ if (ctx->wq->flags & WQ_UNBOUND) From 64496075b02cfca3107999c3f3c641e416799228 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 19 Aug 2026 07:37:38 -0700 Subject: [PATCH 04/16] workqueue: build percpu pwqs through the attrs path Drop the WQ_PERCPU allocation path (alloc_and_link_percpu_pwqs()). Now that alloc_pwq() hands out the static percpu pool for a CPU and the default pwq is optional, the unbound path produces the same result. Point a percpu workqueue at it and drop alloc_and_link_percpu_pwqs(). Nothing is installed in wq->cpu_pwq when the attrs path fails, since apply_wqattrs_commit() does not run, so the enomem cleanup only has the percpu array left to free. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 66 +++++++++++----------------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 10c40fe23e97..8e481609399c 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -516,6 +516,9 @@ static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; /* I: attributes used when instantiating ordered pools on demand */ static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; +/* I: attributes of percpu workqueues, which are backed by the static pools */ +static struct workqueue_attrs *percpu_std_wq_attrs[NR_STD_WORKER_POOLS]; + /* * I: kthread_worker to release pwq's. pwq release needs to be bounced to a * process context while holding a pool lock. Bounce to a dedicated kthread @@ -5541,7 +5544,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs, cpu); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5711,34 +5714,10 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) put_pwq_unlocked(old_pwq); } -static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) -{ - struct pool_workqueue *pwq; - int cpu; - - for_each_possible_cpu(cpu) { - struct worker_pool *pool = get_percpu_pool(wq, cpu); - - pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); - if (!pwq) - return -ENOMEM; - - init_pwq(pwq, wq, pool); - - mutex_lock(&wq->mutex); - link_pwq(pwq); - mutex_unlock(&wq->mutex); - - rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq); - } - - return 0; -} - static int alloc_and_link_pwqs(struct workqueue_struct *wq) { bool highpri = wq->flags & WQ_HIGHPRI; - int cpu, ret; + int ret; lockdep_assert_held(&wq_pool_mutex); @@ -5747,7 +5726,7 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) goto enomem; if (!(wq->flags & WQ_UNBOUND)) { - ret = alloc_and_link_percpu_pwqs(wq); + ret = apply_workqueue_attrs_locked(wq, percpu_std_wq_attrs[highpri]); } else if (wq->flags & __WQ_ORDERED) { struct pool_workqueue *dfl_pwq; @@ -5766,27 +5745,8 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) return 0; enomem: - if (wq->cpu_pwq) { - for_each_possible_cpu(cpu) { - struct pool_workqueue __rcu **slot; - struct pool_workqueue *pwq; - - slot = per_cpu_ptr(wq->cpu_pwq, cpu); - pwq = rcu_access_pointer(*slot); - if (pwq) { - /* - * Unlink pwq from wq->pwqs since link_pwq() - * may have already added it. wq->mutex is not - * needed as the wq has not been published yet. - */ - if (!list_empty(&pwq->pwqs_node)) - list_del_rcu(&pwq->pwqs_node); - kmem_cache_free(pwq_cache, pwq); - } - } - free_percpu(wq->cpu_pwq); - wq->cpu_pwq = NULL; - } + free_percpu(wq->cpu_pwq); + wq->cpu_pwq = NULL; return -ENOMEM; } @@ -6030,10 +5990,10 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, * flushing the pwq_release_worker ensures that the pwq_release_workfn() * completes before calling kfree(wq). */ - if (wq->flags & WQ_UNBOUND) { + if (pwq_release_worker) kthread_flush_worker(pwq_release_worker); + if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); - } err_free_wq: free_workqueue_attrs(wq->attrs); kfree(wq); @@ -8202,7 +8162,7 @@ void __init workqueue_init_early(void) init_cpu_worker_pool(pool, cpu, std_nice[i++]); } - /* create default unbound and ordered wq attrs */ + /* create default unbound, ordered and percpu wq attrs */ for (i = 0; i < NR_STD_WORKER_POOLS; i++) { struct workqueue_attrs *attrs; @@ -8218,6 +8178,10 @@ void __init workqueue_init_early(void) attrs->nice = std_nice[i]; attrs->ordered = true; ordered_wq_attrs[i] = attrs; + + BUG_ON(!(attrs = alloc_workqueue_attrs())); + attrs->nice = std_nice[i]; + percpu_std_wq_attrs[i] = attrs; } system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0); From ab85b68e630e8a4db0271165149589d7702c2532 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 19 Aug 2026 07:37:39 -0700 Subject: [PATCH 05/16] workqueue: rename the pwq slot helpers shared with percpu install_unbound_pwq() fills the pwq slots of a percpu workqueue as well now that alloc_and_link_pwqs() sends it through the attrs path, and unbound_pwq_slot() and unbound_pwq() have always addressed and read those slots for every workqueue, percpu included, from destroy_workqueue(). Their names say otherwise. Rename them to install_pwq(), pwq_slot() and installed_pwq(). Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 8e481609399c..d4d6bb8a23f5 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -742,7 +742,7 @@ static int worker_pool_assign_id(struct worker_pool *pool) } static struct pool_workqueue __rcu ** -unbound_pwq_slot(struct workqueue_struct *wq, int cpu) +pwq_slot(struct workqueue_struct *wq, int cpu) { if (cpu >= 0) return per_cpu_ptr(wq->cpu_pwq, cpu); @@ -751,9 +751,9 @@ unbound_pwq_slot(struct workqueue_struct *wq, int cpu) } /* @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) +static struct pool_workqueue *installed_pwq(struct workqueue_struct *wq, int cpu) { - return rcu_dereference_check(*unbound_pwq_slot(wq, cpu), + return rcu_dereference_check(*pwq_slot(wq, cpu), lockdep_is_held(&wq_pool_mutex) || lockdep_is_held(&wq->mutex)); } @@ -768,7 +768,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) */ static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq) { - return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask; + return installed_pwq(wq, -1)->pool->attrs->__pod_cpumask; } static unsigned int work_color_to_flags(int color) @@ -5459,10 +5459,10 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu) } /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, - int cpu, struct pool_workqueue *pwq) +static struct pool_workqueue *install_pwq(struct workqueue_struct *wq, + int cpu, struct pool_workqueue *pwq) { - struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu); + struct pool_workqueue __rcu **slot = pwq_slot(wq, cpu); struct pool_workqueue *old_pwq; lockdep_assert_held(&wq_pool_mutex); @@ -5586,10 +5586,10 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) /* save the previous pwqs and install the new ones */ for_each_possible_cpu(cpu) - ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, - ctx->pwq_tbl[cpu]); + ctx->pwq_tbl[cpu] = install_pwq(ctx->wq, cpu, + ctx->pwq_tbl[cpu]); if (ctx->dfl_pwq) - ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); + ctx->dfl_pwq = install_pwq(ctx->wq, -1, ctx->dfl_pwq); /* update node_nr_active->max, which only unbound workqueues have */ if (ctx->wq->flags & WQ_UNBOUND) @@ -5686,7 +5686,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* nothing to do if the target cpumask matches the current pwq */ wq_calc_pod_cpumask(target_attrs, cpu); - if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs)) + if (wqattrs_equal(target_attrs, installed_pwq(wq, cpu)->pool->attrs)) return; /* create a new pwq */ @@ -5699,16 +5699,16 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* Install the new pwq. */ mutex_lock(&wq->mutex); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); goto out_unlock; use_dfl_pwq: mutex_lock(&wq->mutex); - pwq = unbound_pwq(wq, -1); + pwq = installed_pwq(wq, -1); raw_spin_lock_irq(&pwq->pool->lock); get_pwq(pwq); raw_spin_unlock_irq(&pwq->pool->lock); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); out_unlock: mutex_unlock(&wq->mutex); put_pwq_unlocked(old_pwq); @@ -6182,12 +6182,12 @@ void destroy_workqueue(struct workqueue_struct *wq) rcu_read_lock(); for_each_possible_cpu(cpu) { - put_pwq_unlocked(unbound_pwq(wq, cpu)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL); + put_pwq_unlocked(installed_pwq(wq, cpu)); + RCU_INIT_POINTER(*pwq_slot(wq, cpu), NULL); } - put_pwq_unlocked(unbound_pwq(wq, -1)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); + put_pwq_unlocked(installed_pwq(wq, -1)); + RCU_INIT_POINTER(*pwq_slot(wq, -1), NULL); rcu_read_unlock(); } From a4d1cdc4356de4325b0a9c6fb09afa6e121407b5 Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Fri, 4 Sep 2026 15:34:45 -0400 Subject: [PATCH 06/16] workqueue: Add workqueue_cpu_intensive tracepoint When a concurrency-managed per-CPU work item runs continuously without sleeping for longer than wq_cpu_intensive_thresh_us, wq_worker_tick() marks the worker as WORKER_CPU_INTENSIVE and kicks it out of concurrency management so that pending work items on the pool are not starved. While CONFIG_WQ_CPU_INTENSIVE_REPORT logs rate-limited warnings and pwq->stats[PWQ_STAT_CPU_INTENSIVE] maintains a cumulative counter, there is currently no tracepoint emitted at the moment of this transition. Therefore, add the workqueue_cpu_intensive tracepoint, recording the work_struct pointer and callback function pointer, workqueue name, executing CPU, and the runtime duration consumed in microseconds. This enables eBPF profilers, bpftrace, and Ftrace to immediately detect and attribute CPU-hogging work items in real time. Signed-off-by: Aaron Tomlin Signed-off-by: Tejun Heo --- include/trace/events/workqueue.h | 40 ++++++++++++++++++++++++++++++++ kernel/workqueue.c | 7 ++++++ 2 files changed, 47 insertions(+) diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h index b0de2bc9ed52..d2ad38f93a83 100644 --- a/include/trace/events/workqueue.h +++ b/include/trace/events/workqueue.h @@ -126,7 +126,47 @@ TRACE_EVENT(workqueue_execute_end, TP_printk("work struct %p: function %ps", __entry->work, __entry->function) ); +/** + * workqueue_cpu_intensive - called when a work item exceeds cpu_intensive threshold + * @pwq: pointer to struct pool_workqueue + * @work: pointer to struct work_struct + * @function: pointer to worker function + * @duration_us: CPU time consumed in microseconds + * + * This event occurs when a concurrency-managed work item runs for longer + * than wq_cpu_intensive_thresh_us without sleeping and is excluded from + * concurrency management to prevent stalling other work items. + */ +TRACE_EVENT(workqueue_cpu_intensive, + + TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work, + work_func_t function, u64 duration_us), + + TP_ARGS(pwq, work, function, duration_us), + + TP_STRUCT__entry( + __field( void *, work ) + __field( void *, function ) + __string( workqueue, pwq->wq->name ) + __field( int, cpu ) + __field( u64, duration_us ) + ), + + TP_fast_assign( + __entry->work = work; + __entry->function = function; + __assign_str(workqueue); + __entry->cpu = pwq->pool->cpu; + __entry->duration_us = duration_us; + ), + + TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d duration_us=%llu", + __entry->work, __entry->function, __get_str(workqueue), + __entry->cpu, __entry->duration_us) +); + #endif /* _TRACE_WORKQUEUE_H */ /* This part must be outside protection */ #include + diff --git a/kernel/workqueue.c b/kernel/workqueue.c index d4d6bb8a23f5..22fb41b6ec26 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -56,6 +56,7 @@ #include #include #include +#include #include "workqueue_internal.h" @@ -1535,6 +1536,7 @@ void wq_worker_tick(struct task_struct *task) struct worker *worker = kthread_data(task); struct pool_workqueue *pwq = worker->current_pwq; struct worker_pool *pool = worker->pool; + u64 delta; if (!pwq) return; @@ -1575,6 +1577,11 @@ void wq_worker_tick(struct task_struct *task) pwq->stats[PWQ_STAT_CM_WAKEUP]++; raw_spin_unlock(&pool->lock); + + delta = READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at; + trace_workqueue_cpu_intensive(pwq, worker->current_work, + worker->current_func, + div_u64(delta, NSEC_PER_USEC)); } /** From a53b50e21af2eb8d4e74c8f84e409692a3332588 Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Fri, 4 Sep 2026 15:34:46 -0400 Subject: [PATCH 07/16] workqueue: Add workqueue_mayday and workqueue_rescued tracepoints When a worker pool fails to create a new worker thread (e.g., typically under severe memory pressure where memory reclaim paths depend on pending work items), send_mayday() signals distress to the workqueue's rescuer thread. The rescuer then takes over processing the pending work items via assign_rescuer_work(). While pwq->stats[PWQ_STAT_MAYDAY] and pwq->stats[PWQ_STAT_RESCUED] track these occurrences cumulatively, there is currently no event-driven mechanism to observe exactly when mayday distress occurs or which work items require rescue. Add two new tracepoints namely workqueue_mayday and workqueue_rescued to make distress and rescuer execution easily observable (e.g., via Ftrace or eBPF). Signed-off-by: Aaron Tomlin Signed-off-by: Tejun Heo --- include/trace/events/workqueue.h | 67 ++++++++++++++++++++++++++++++++ kernel/workqueue.c | 2 + 2 files changed, 69 insertions(+) diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h index d2ad38f93a83..eb97d4dbb5b4 100644 --- a/include/trace/events/workqueue.h +++ b/include/trace/events/workqueue.h @@ -165,6 +165,73 @@ TRACE_EVENT(workqueue_cpu_intensive, __entry->cpu, __entry->duration_us) ); +/** + * workqueue_mayday - called when a pool_workqueue sends mayday to rescuer + * @pwq: pointer to struct pool_workqueue + * + * This event occurs when a worker pool fails to create a new worker and + * requests the workqueue's rescuer thread to process pending works. + */ +TRACE_EVENT(workqueue_mayday, + + TP_PROTO(struct pool_workqueue *pwq), + + TP_ARGS(pwq), + + TP_STRUCT__entry( + __string( workqueue, pwq->wq->name ) + __field( int, pool_id ) + __field( int, cpu ) + __field( int, nr_active ) + ), + + TP_fast_assign( + __assign_str(workqueue); + __entry->pool_id = pwq->pool->id; + __entry->cpu = pwq->pool->cpu; + __entry->nr_active = pwq->nr_active; + ), + + TP_printk("workqueue=%s pool_id=%d cpu=%d nr_active=%d", + __get_str(workqueue), __entry->pool_id, __entry->cpu, + __entry->nr_active) +); + +/** + * workqueue_rescued - called when a work item is assigned to a rescuer + * @pwq: pointer to struct pool_workqueue + * @work: pointer to struct work_struct + * @function: pointer to worker function + * + * This event occurs when a work item is claimed by a rescuer thread + * to guarantee forward progress. + */ +TRACE_EVENT(workqueue_rescued, + + TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work, + work_func_t function), + + TP_ARGS(pwq, work, function), + + TP_STRUCT__entry( + __field( void *, work ) + __field( void *, function ) + __string( workqueue, pwq->wq->name ) + __field( int, cpu ) + ), + + TP_fast_assign( + __entry->work = work; + __entry->function = function; + __assign_str(workqueue); + __entry->cpu = pwq->pool->cpu; + ), + + TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d", + __entry->work, __entry->function, __get_str(workqueue), + __entry->cpu) +); + #endif /* _TRACE_WORKQUEUE_H */ /* This part must be outside protection */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 22fb41b6ec26..53bb9ad56f49 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3127,6 +3127,7 @@ static void send_mayday(struct pool_workqueue *pwq) list_add_tail(&pwq->mayday_node, &wq->maydays); wake_up_process(wq->rescuer->task); pwq->stats[PWQ_STAT_MAYDAY]++; + trace_workqueue_mayday(pwq); } } @@ -3633,6 +3634,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { pwq->stats[PWQ_STAT_RESCUED]++; + trace_workqueue_rescued(pwq, work, work->func); /* put the cursor for next search */ list_move_tail(&cursor->entry, &n->entry); return true; From 8bbd858c4710ff90491365c09209d9d6f70ced77 Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Fri, 4 Sep 2026 15:34:47 -0400 Subject: [PATCH 08/16] workqueue: Add workqueue_bh_budget_yield tracepoint Bottom-Half (BH) workqueues execute work items in softirq context. To prevent softirqs from starving user and kernel threads, bh_worker() enforces execution limits (i.e., BH_WORKER_JIFFIES and BH_WORKER_RESTARTS). When keep_working() is still true but either the time slice or restart count is exhausted, bh_worker() yields execution and re-raises the softirq via kick_bh_pool(). Currently, there is no observability into when a BH worker hits these limits and is forced to yield. Add the workqueue_bh_budget_yield tracepoint, emitted when bh_worker() exits the processing loop with pending work items remaining. It records, the worker pool ID, pool CPU, number of loop restarts consumed, a boolean flag indicating whether the yield was due to a time slice timeout, and a boolean flag indicating whether this is a high-priority BH pool. Signed-off-by: Aaron Tomlin Signed-off-by: Tejun Heo --- include/trace/events/workqueue.h | 39 ++++++++++++++++++++++++++++++++ kernel/workqueue.c | 26 +++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h index eb97d4dbb5b4..5b820ff8f81a 100644 --- a/include/trace/events/workqueue.h +++ b/include/trace/events/workqueue.h @@ -9,6 +9,7 @@ #include struct pool_workqueue; +struct worker_pool; /** * workqueue_queue_work - called when a work gets queued @@ -232,6 +233,44 @@ TRACE_EVENT(workqueue_rescued, __entry->cpu) ); +/** + * workqueue_bh_budget_yield - called when a BH worker yields due to budget exhaustion + * @pool: pointer to struct worker_pool + * @restarts: number of restarts executed + * @timeout: whether execution hit the time limit (BH_WORKER_JIFFIES) + * @highpri: whether this is a high-priority BH pool + * + * This event occurs when a bottom-half (BH) worker pool running in softirq + * context exhausts its execution time slice or restart limit and must yield + * execution. + */ +TRACE_EVENT(workqueue_bh_budget_yield, + + TP_PROTO(struct worker_pool *pool, int restarts, bool timeout, bool highpri), + + TP_ARGS(pool, restarts, timeout, highpri), + + TP_STRUCT__entry( + __field( int, pool_id ) + __field( int, cpu ) + __field( int, restarts ) + __field( bool, timeout ) + __field( bool, highpri ) + ), + + TP_fast_assign( + __entry->pool_id = pool->id; + __entry->cpu = pool->cpu; + __entry->restarts = restarts; + __entry->timeout = timeout; + __entry->highpri = highpri; + ), + + TP_printk("pool_id=%d cpu=%d restarts=%d timeout=%d highpri=%d", + __entry->pool_id, __entry->cpu, __entry->restarts, + __entry->timeout, __entry->highpri) +); + #endif /* _TRACE_WORKQUEUE_H */ /* This part must be outside protection */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 53bb9ad56f49..5b70489a1e65 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3770,6 +3770,9 @@ static void bh_worker(struct worker *worker) struct worker_pool *pool = worker->pool; int nr_restarts = BH_WORKER_RESTARTS; unsigned long end = jiffies + BH_WORKER_JIFFIES; + bool budget_exhausted = false; + bool timeout = false; + int restarts = 0; worker_lock_callback(pool); raw_spin_lock_irq(&pool->lock); @@ -3792,8 +3795,23 @@ static void bh_worker(struct worker *worker) if (assign_work(work, worker, NULL)) process_scheduled_works(worker); - } while (keep_working(pool) && - --nr_restarts && time_before(jiffies, end)); + + if (!keep_working(pool)) + break; + + if (!--nr_restarts) { + budget_exhausted = true; + break; + } + + if (!time_before(jiffies, end)) { + budget_exhausted = true; + timeout = true; + break; + } + + restarts++; + } while (1); worker_set_flags(worker, WORKER_PREP); done: @@ -3801,6 +3819,10 @@ static void bh_worker(struct worker *worker) kick_pool(pool); raw_spin_unlock_irq(&pool->lock); worker_unlock_callback(pool); + + if (budget_exhausted) + trace_workqueue_bh_budget_yield(pool, restarts, timeout, + pool->attrs->nice == HIGHPRI_NICE_LEVEL); } /* From 1298c43b673274920b26e77d3df55c18c3b82f90 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 11:53:38 +0530 Subject: [PATCH 09/16] workqueue: Fix typo "upto" in comment Correct "upto" to "up to", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Tejun Heo --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5b70489a1e65..79bb8482bc8d 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -8258,7 +8258,7 @@ static void __init wq_cpu_intensive_thresh_init(void) * lot slower CPUs including microcontrollers where the threshold is way * too low. * - * Let's scale up the threshold upto 1 second if BogoMips is below 4000. + * Let's scale up the threshold up to 1 second if BogoMips is below 4000. * This is by no means accurate but it doesn't have to be. The mechanism * is still useful even when the threshold is fully scaled up. Also, as * the reports would usually be applicable to everyone, some machines From 84a42d3829f0088967411106b81e942a67d0b53c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 1 Sep 2026 11:09:27 -1000 Subject: [PATCH 10/16] workqueue: Add for_each_node_with_fallback() Per-node arrays in workqueue carry an extra slot at nr_node_ids for NUMA_NO_NODE, and iterating them open-codes for_each_node() plus separate handling of that slot. Add for_each_node_with_fallback() and convert alloc/free_node_nr_active(). Signed-off-by: Tejun Heo --- kernel/workqueue.c | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 79bb8482bc8d..298f22c4caad 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -623,6 +623,26 @@ static void show_one_worker_pool(struct worker_pool *pool); list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \ lockdep_is_held(&(wq->mutex))) +/* + * Per-node arrays in this file carry an extra slot at nr_node_ids serving + * NUMA_NO_NODE. Return the slot after all nodes. + */ +static int next_node_with_fallback(int node) +{ + if (node >= nr_node_ids) + return nr_node_ids + 1; + + node = next_node(node, node_possible_map); + if (node >= nr_node_ids) + return nr_node_ids; + return node; +} + +#define for_each_node_with_fallback(node) \ + for ((node) = first_node(node_possible_map); \ + (node) <= nr_node_ids; \ + (node) = next_node_with_fallback(node)) + #ifdef CONFIG_DEBUG_OBJECTS_WORK static const struct debug_obj_descr work_debug_descr; @@ -5083,13 +5103,10 @@ static void free_node_nr_active(struct wq_node_nr_active **nna_ar) { int node; - for_each_node(node) { + for_each_node_with_fallback(node) { kfree(nna_ar[node]); nna_ar[node] = NULL; } - - kfree(nna_ar[nr_node_ids]); - nna_ar[nr_node_ids] = NULL; } static void init_node_nr_active(struct wq_node_nr_active *nna) @@ -5109,21 +5126,15 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar) struct wq_node_nr_active *nna; int node; - for_each_node(node) { - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node); + for_each_node_with_fallback(node) { + nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); if (!nna) goto err_free; init_node_nr_active(nna); nna_ar[node] = nna; } - /* [nr_node_ids] is used as the fallback */ - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE); - if (!nna) - goto err_free; - init_node_nr_active(nna); - nna_ar[nr_node_ids] = nna; - return 0; err_free: From 31e33de1f4e159c7c6c44001fbd3398d53b0311d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 1 Sep 2026 11:09:28 -1000 Subject: [PATCH 11/16] workqueue: Maintain pwq->total_in_flight pwq_busy() scans all of pwq->nr_in_flight[] to tell whether anything is in flight. Maintain the sum in pwq->total_in_flight, which fits in existing padding, and test that instead. Signed-off-by: Tejun Heo --- kernel/workqueue.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 298f22c4caad..322de4f74e2f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -275,6 +275,7 @@ struct pool_workqueue { int work_color; /* L: current color */ int flush_color; /* L: flushing color */ int refcnt; /* L: reference count */ + int total_in_flight; /* L: sum of nr_in_flight[] */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ bool plugged; /* L: execution suspended */ @@ -2089,6 +2090,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) node_activate_pending_pwq(nna, pool); } +/** + * pwq_inc_nr_in_flight - increment pwq's nr_in_flight + * @pwq: pwq of interest + * @work_color: color of the work item being queued + * + * A work item or a barrier with @work_color is being queued to @pwq. + * + * CONTEXT: + * raw_spin_lock_irq(pool->lock). + */ +static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color) +{ + pwq->nr_in_flight[work_color]++; + pwq->total_in_flight++; +} + /** * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight * @pwq: pwq of interest @@ -2113,6 +2130,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_ pwq_dec_nr_active(pwq); pwq->nr_in_flight[color]--; + pwq->total_in_flight--; /* is flush in progress and are we at the flushing tip? */ if (likely(pwq->flush_color != color)) @@ -2462,7 +2480,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, if (WARN_ON(!list_empty(&work->entry))) goto out; - pwq->nr_in_flight[pwq->work_color]++; + pwq_inc_nr_in_flight(pwq, pwq->work_color); work_flags = work_color_to_flags(pwq->work_color); /* @@ -4062,7 +4080,7 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, __set_bit(WORK_STRUCT_LINKED_BIT, bits); } - pwq->nr_in_flight[work_color]++; + pwq_inc_nr_in_flight(pwq, work_color); work_flags |= work_color_to_flags(work_color); insert_work(pwq, &barr->work, head, work_flags); @@ -6129,11 +6147,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map); static bool pwq_busy(struct pool_workqueue *pwq) { - int i; - - for (i = 0; i < WORK_NR_COLORS; i++) - if (pwq->nr_in_flight[i]) - return true; + if (pwq->total_in_flight) + return true; if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) return true; From f1639cde5f5c1bee25498ef988227ee8cf5ddb04 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 1 Sep 2026 11:09:29 -1000 Subject: [PATCH 12/16] workqueue: Make flush_workqueue() visit only pwqs active since the last flush 636b927eba5b ("workqueue: Make unbound workqueues to use per-cpu pool_workqueues") fixed unbound workqueue scalability on large machines but made flush_workqueue() walk one pwq per possible CPU, cycling each pool lock, up to three times per flush. 85f0d8e39aff ("workqueue: Reduce expensive locks for unbound workqueue") coalesced same-pool locks for unbound workqueues but the walk remains. Yao Kai reported the XFS CIL workqueue, flushed on every log force, spending up to 64us per flush in the walk on a 128-CPU machine. Idle pwqs can't just be skipped: the lock-and-advance of every pwq's work color is what keeps a concurrently queued work item from being stamped with a retired color unseen by the flusher, which would let a later flush return before it finishes. Track active pwqs instead. Each workqueue gets a wq_flush_pnode per node with a lock, a mirror of wq->work_color and a list of pwqs. Queueing to an off-list pwq syncs pwq->work_color from the mirror and adds it under fpn->lock, and flush_workqueue_prep_pwqs() advances the mirror and splices the list in one fpn->lock section per node before visiting the pwqs under pool->lock as before. That replaces the fence: a racing queueing either gets its pwq on the list before the splice or stamps the advanced color. A pwq stays on the list until a visit finds nothing in flight, so a cascade arming an older color still finds it, barriers need no separate add as the work item they follow keeps the pwq on the list, and pwq_release_workfn() removes a released pwq under wq->mutex. On a 192-CPU 2-node machine, flushing an idle per-cpu workqueue goes from 40k to 4.2M per second and an idle unbound one from 450k to 4.3M. 16 threads each queueing a work item and flushing, go from 30k to 110k flushes per second on a per-cpu workqueue and 60k to 180k on an unbound one. Dense flushes with every pwq active are unchanged on per-cpu workqueues and 15-20% slower on unbound ones. The queue path is unchanged. Reported-by: Yao Kai Link: https://lore.kernel.org/all/0a030145-c108-4365-ba2d-ac1973a1e352@huawei.com/ Signed-off-by: Tejun Heo --- kernel/workqueue.c | 241 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 195 insertions(+), 46 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 322de4f74e2f..f868011e291f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -162,6 +162,9 @@ struct llc_shard_layout { * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for * reads. * + * FN: wq_flush_pnode->lock protected. list_empty(&pwq->flush_node) can also be + * tested with only pool->lock held. + * * K: Only modified by worker while holding pool->lock. Can be safely read by * self, while holding pool->lock or from IRQ context if %current is the * kworker. @@ -278,6 +281,7 @@ struct pool_workqueue { int total_in_flight; /* L: sum of nr_in_flight[] */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ + struct list_head flush_node; /* FN: node on wq_flush_pnode->pwqs */ bool plugged; /* L: execution suspended */ /* @@ -346,6 +350,21 @@ struct wq_node_nr_active { struct list_head pending_pwqs; /* LN: pwqs with inactive works */ }; +/* + * Per-node list of pwqs active since the last flush so that flushes visit only + * those. One per possible node plus the NUMA_NO_NODE fallback at nr_node_ids, + * each allocated on its node. A pwq is added when a work item is queued to it + * while off the list and removed by a flush which finds it with nothing in + * flight. ->work_color mirrors wq->work_color. Reading it and adding the pwq in + * one ->lock section keeps queueing coherent against flushing: a work item is + * either stamped with the new color or its pwq is visible to the flusher. + */ +struct wq_flush_pnode { + raw_spinlock_t lock; /* nests inside pool locks */ + int work_color; /* FN: mirrors wq->work_color */ + struct list_head pwqs; /* FN: pwqs active since last flush */ +}; + /* * The externally visible workqueue. It relays the issued work items to * the appropriate worker_pool through its pool_workqueues. @@ -361,6 +380,7 @@ struct workqueue_struct { struct wq_flusher *first_flusher; /* WQ: first flusher */ struct list_head flusher_queue; /* WQ: flush waiters */ struct list_head flusher_overflow; /* WQ: flush overflow list */ + struct wq_flush_pnode **flush_pnodes; /* I: per-node flush membership */ struct list_head maydays; /* MD: pwqs requesting rescue */ struct worker *rescuer; /* MD: rescue worker */ @@ -1665,6 +1685,22 @@ static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, return wq->node_nr_active[node]; } +/** + * wq_flush_pnode - Determine wq_flush_pnode to use + * @wq: workqueue of interest + * @node: NUMA node, can be %NUMA_NO_NODE + * + * Return @wq's wq_flush_pnode for @node, the nr_node_ids fallback if @node is + * %NUMA_NO_NODE. + */ +static struct wq_flush_pnode *wq_flush_pnode(struct workqueue_struct *wq, int node) +{ + if (node == NUMA_NO_NODE) + node = nr_node_ids; + + return wq->flush_pnodes[node]; +} + /** * wq_update_node_max_active - Update per-node max_actives to use * @wq: workqueue to update @@ -2480,6 +2516,20 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, if (WARN_ON(!list_empty(&work->entry))) goto out; + /* + * Add @pwq to its wq_flush_pnode list if off it. Syncing ->work_color + * in the same fpn->lock section is what keeps a concurrent flush from + * missing @work, see wq_flush_pnode. + */ + if (unlikely(list_empty(&pwq->flush_node))) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock(&fpn->lock); + pwq->work_color = fpn->work_color; + list_add_tail(&pwq->flush_node, &fpn->pwqs); + raw_spin_unlock(&fpn->lock); + } + pwq_inc_nr_in_flight(pwq, pwq->work_color); work_flags = work_color_to_flags(pwq->work_color); @@ -4080,6 +4130,14 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, __set_bit(WORK_STRUCT_LINKED_BIT, bits); } + /* + * Flushes must wait for barriers too as a barrier pins @pwq and a + * worker until it runs and destroy_workqueue()'s drain relies on that. + * @target being in flight keeps @pwq on its wq_flush_pnode list, so no + * need to add it here. + */ + WARN_ON_ONCE(list_empty(&pwq->flush_node)); + pwq_inc_nr_in_flight(pwq, work_color); work_flags |= work_color_to_flags(work_color); @@ -4092,23 +4150,23 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * - * Prepare pwqs for workqueue flushing. + * Prepare pwqs for workqueue flushing. Only the pwqs on @wq's wq_flush_pnode + * lists, which have been active since the last flush, are visited. * - * If @flush_color is non-negative, flush_color on all pwqs should be - * -1. If no pwq has in-flight commands at the specified color, all - * pwq->flush_color's stay at -1 and %false is returned. If any pwq - * has in flight commands, its pwq->flush_color is set to - * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq - * wakeup logic is armed and %true is returned. + * If @flush_color >= 0, flush_color on all visited pwqs should be -1. If no + * visited pwq has in-flight work items at the specified color, all + * pwq->flush_color's stay at -1 and return %false. If any visited pwq has + * in-flight work items, set its pwq->flush_color to @flush_color, update + * @wq->nr_pwqs_to_flush accordingly, arm pwq wakeup logic and return %true. * - * The caller should have initialized @wq->first_flusher prior to - * calling this function with non-negative @flush_color. If - * @flush_color is negative, no flush color update is done and %false - * is returned. + * The caller should have initialized @wq->first_flusher prior to calling this + * function with non-negative @flush_color. If @flush_color < 0, no flush color + * update is done and %false is returned. * - * If @work_color is non-negative, all pwqs should have the same - * work_color which is previous to @work_color and all will be - * advanced to @work_color. + * If @work_color >= 0, all visited pwqs and every wq_flush_pnode's work color + * mirror are advanced to @work_color. The mirrors of nodes with no active pwqs + * must be advanced too as a pwq added to a list later stamps the color it reads + * from the mirror. * * CONTEXT: * mutex_lock(wq->mutex). @@ -4120,50 +4178,81 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, int flush_color, int work_color) { + struct pool_workqueue *pwq, *next; bool wait = false; - struct pool_workqueue *pwq; - struct worker_pool *current_pool = NULL; + int node; if (flush_color >= 0) { WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); atomic_set(&wq->nr_pwqs_to_flush, 1); } - /* - * For unbound workqueue, pwqs will map to only a few pools. - * Most of the time, pwqs within the same pool will be linked - * sequentially to wq->pwqs by cpu index. So in the majority - * of pwq iters, the pool is the same, only doing lock/unlock - * if the pool has changed. This can largely reduce expensive - * lock operations. - */ - for_each_pwq(pwq, wq) { - if (current_pool != pwq->pool) { - if (likely(current_pool)) - raw_spin_unlock_irq(¤t_pool->lock); - current_pool = pwq->pool; - raw_spin_lock_irq(¤t_pool->lock); - } - - if (flush_color >= 0) { - WARN_ON_ONCE(pwq->flush_color != -1); - - if (pwq->nr_in_flight[flush_color]) { - pwq->flush_color = flush_color; - atomic_inc(&wq->nr_pwqs_to_flush); - wait = true; - } - } + for_each_node_with_fallback(node) { + struct wq_flush_pnode *fpn = wq->flush_pnodes[node]; + struct worker_pool *current_pool = NULL; + LIST_HEAD(to_visit); + LIST_HEAD(to_keep); + /* + * Advance the node's color mirror and take its active pwqs. + * Work items queued to off-list pwqs afterwards carry the new + * color and don't need visiting. + */ + raw_spin_lock_irq(&fpn->lock); if (work_color >= 0) { - WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); - pwq->work_color = work_color; + WARN_ON_ONCE(work_color != work_next_color(fpn->work_color)); + fpn->work_color = work_color; + } + list_splice_init(&fpn->pwqs, &to_visit); + raw_spin_unlock_irq(&fpn->lock); + + /* + * Unbound workqueues map multiple pwqs to one pool. Only cycle + * pool->lock when the pool changes. + */ + list_for_each_entry_safe(pwq, next, &to_visit, flush_node) { + if (current_pool != pwq->pool) { + if (likely(current_pool)) + raw_spin_unlock_irq(¤t_pool->lock); + current_pool = pwq->pool; + raw_spin_lock_irq(¤t_pool->lock); + } + + if (flush_color >= 0) { + WARN_ON_ONCE(pwq->flush_color != -1); + + if (pwq->nr_in_flight[flush_color]) { + pwq->flush_color = flush_color; + atomic_inc(&wq->nr_pwqs_to_flush); + wait = true; + } + } + + if (work_color >= 0) { + WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); + pwq->work_color = work_color; + } + + /* + * A cascade can arm a color retired several advances + * ago, so keep a pwq listed while any color has work + * items in flight. + */ + if (pwq->total_in_flight) + list_move_tail(&pwq->flush_node, &to_keep); + else + list_del_init(&pwq->flush_node); } - } + if (current_pool) + raw_spin_unlock_irq(¤t_pool->lock); - if (current_pool) - raw_spin_unlock_irq(¤t_pool->lock); + if (!list_empty(&to_keep)) { + raw_spin_lock_irq(&fpn->lock); + list_splice_tail(&to_keep, &fpn->pwqs); + raw_spin_unlock_irq(&fpn->lock); + } + } if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) complete(&wq->first_flusher->done); @@ -5160,6 +5249,47 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar) return -ENOMEM; } +static void free_flush_pnodes(struct workqueue_struct *wq) +{ + int node; + + if (!wq->flush_pnodes) + return; + + for_each_node_with_fallback(node) + kfree(wq->flush_pnodes[node]); + kfree(wq->flush_pnodes); + wq->flush_pnodes = NULL; +} + +static int alloc_flush_pnodes(struct workqueue_struct *wq) +{ + struct wq_flush_pnode *fpn; + int node; + + wq->flush_pnodes = kcalloc(nr_node_ids + 1, sizeof(*wq->flush_pnodes), GFP_KERNEL); + if (!wq->flush_pnodes) + return -ENOMEM; + + for_each_node_with_fallback(node) { + fpn = kzalloc_node(sizeof(*fpn), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); + if (!fpn) + goto err_free; + + raw_spin_lock_init(&fpn->lock); + fpn->work_color = wq->work_color; + INIT_LIST_HEAD(&fpn->pwqs); + wq->flush_pnodes[node] = fpn; + } + + return 0; + +err_free: + free_flush_pnodes(wq); + return -ENOMEM; +} + static void rcu_free_wq(struct rcu_head *rcu) { struct workqueue_struct *wq = @@ -5168,6 +5298,7 @@ static void rcu_free_wq(struct rcu_head *rcu) if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); + free_flush_pnodes(wq); wq_free_lockdep(wq); free_percpu(wq->cpu_pwq); free_workqueue_attrs(wq->attrs); @@ -5348,6 +5479,19 @@ static void pwq_release_workfn(struct kthread_work *work) list_del_rcu(&pwq->pwqs_node); is_last = list_empty(&wq->pwqs); + /* + * An idle pwq can linger on its wq_flush_pnode list until the + * next flush. wq->mutex excludes flushers, which never drop it + * with pwqs on their private lists. + */ + if (!list_empty(&pwq->flush_node)) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock_irq(&fpn->lock); + list_del_init(&pwq->flush_node); + raw_spin_unlock_irq(&fpn->lock); + } + /* * For ordered workqueue with a plugged dfl_pwq, restart it now. */ @@ -5398,6 +5542,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, pwq->refcnt = 1; INIT_LIST_HEAD(&pwq->inactive_works); INIT_LIST_HEAD(&pwq->pending_node); + INIT_LIST_HEAD(&pwq->flush_node); INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); @@ -6013,6 +6158,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, INIT_LIST_HEAD(&wq->list); + if (alloc_flush_pnodes(wq) < 0) + goto err_free_wq; + if (flags & WQ_UNBOUND) { if (alloc_node_nr_active(wq->node_nr_active) < 0) goto err_free_wq; @@ -6056,6 +6204,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, free_node_nr_active(wq->node_nr_active); err_free_wq: free_workqueue_attrs(wq->attrs); + free_flush_pnodes(wq); kfree(wq); return NULL; err_unlock_destroy: From 27db9dd7f84f3a7704596455b508611da3d6269d Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 10 Sep 2026 07:54:12 -0700 Subject: [PATCH 13/16] workqueue: Give percpu workqueues their own max_active wq->max_active means two different things, depending on how the workqueue is backed. On a concurrency managed per-cpu pool it caps pwq->nr_active, so it is a per-cpu limit. On an unbound pool it feeds wq_node_nr_active() and is shared by the whole workqueue. Tejun once said [1]: It kinda sucks that max_active's meaning is different across the boundary tho. Maybe percpu max_active should be separate into its own field, idk. Add wq->percpu_max_active for the first meaning and leave max_active as the system wide one. wq_adjust_max_active() updates whichever of the two applies, and min_active stays with max_active as only the unbound side uses it. Suggested-by: Tejun Heo Link: https://lore.kernel.org/all/amESSqf0TMmzhFGz@slm.duckdns.org/ [1] Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f868011e291f..8232a60e107c 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -389,6 +389,7 @@ struct workqueue_struct { /* See alloc_workqueue() function comment for info on min/max_active */ int max_active; /* WO: max active works */ + int percpu_max_active; /* WO: max active works per cpu */ int min_active; /* WO: min active works */ int saved_max_active; /* WQ: saved max_active */ int saved_min_active; /* WQ: saved min_active */ @@ -1855,10 +1856,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) /* * A concurrency-managed per-cpu pool accounts nr_active per pwq, so - * pwq->nr_active against wq->max_active is sufficient. + * pwq->nr_active against wq->percpu_max_active is sufficient. */ if (is_percpu_pool(pool)) { - obtained = pwq->nr_active < READ_ONCE(wq->max_active); + obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active); goto out; } @@ -6016,9 +6017,9 @@ static int init_rescuer(struct workqueue_struct *wq) * wq_adjust_max_active - update a wq's max_active to the current setting * @wq: target workqueue * - * If @wq isn't freezing, set @wq->max_active to the saved_max_active and - * activate inactive work items accordingly. If @wq is freezing, clear - * @wq->max_active to zero. + * If @wq isn't freezing, set the limit that applies to @wq's backing to the + * saved_max_active and activate inactive work items accordingly. If @wq is + * freezing, clear it to zero. */ static void wq_adjust_max_active(struct workqueue_struct *wq) { @@ -6035,20 +6036,25 @@ static void wq_adjust_max_active(struct workqueue_struct *wq) new_min = wq->saved_min_active; } - if (wq->max_active == new_max && wq->min_active == new_min) - return; - /* - * Update @wq->max/min_active and then kick inactive work items if more - * active work items are allowed. This doesn't break work item ordering + * Update the limit and then kick inactive work items if more active + * work items are allowed. This doesn't break work item ordering * because new work items are always queued behind existing inactive * work items if there are any. */ - WRITE_ONCE(wq->max_active, new_max); - WRITE_ONCE(wq->min_active, new_min); + if (wq->flags & WQ_UNBOUND) { + if (wq->max_active == new_max && wq->min_active == new_min) + return; - if (wq->flags & WQ_UNBOUND) + WRITE_ONCE(wq->max_active, new_max); + WRITE_ONCE(wq->min_active, new_min); wq_update_node_max_active(wq, -1); + } else { + if (wq->percpu_max_active == new_max) + return; + + WRITE_ONCE(wq->percpu_max_active, new_max); + } if (new_max == 0) return; @@ -6145,10 +6151,14 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, /* init wq */ wq->flags = flags; - wq->max_active = max_active; - wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); - wq->saved_max_active = wq->max_active; - wq->saved_min_active = wq->min_active; + if (flags & WQ_UNBOUND) { + wq->max_active = max_active; + wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); + wq->saved_min_active = wq->min_active; + } else { + wq->percpu_max_active = max_active; + } + wq->saved_max_active = max_active; mutex_init(&wq->mutex); atomic_set(&wq->nr_pwqs_to_flush, 0); INIT_LIST_HEAD(&wq->pwqs); From d8c2ce77cd09077ec57c4bed5678f76425157ccd Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 10 Sep 2026 07:54:13 -0700 Subject: [PATCH 14/16] workqueue: Derive the standard attrs from the workqueue flags alloc_and_link_pwqs() picks one of three static attrs arrays indexed by WQ_HIGHPRI, and branches to a separate apply_workqueue_attrs_locked() call for percpu, ordered and unbound workqueues. The arrays differ only in the nice level and the ordered bit, which the flags already say. Build the attrs from the flags instead. alloc_wq_std_attrs() sets nice for WQ_HIGHPRI and ordered for __WQ_ORDERED, which collapses the three branches into one call and drops the three arrays along with their workqueue_init_early() loop. The ordering WARN now sits behind its own __WQ_ORDERED test rather than the unbound branch. __WQ_ORDERED is internal and every macro that sets it pairs it with WQ_UNBOUND, so the WARN covers the same workqueues as before. The attrs are allocated per creation now. apply_wqattrs_prepare() already allocates one on the same path, including in workqueue_init_early(), so that is not a new constraint, and it only copies what it is given, so the caller can free them once apply_workqueue_attrs_locked() returns. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 80 ++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 8232a60e107c..39187f424c34 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -533,15 +533,6 @@ static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */ /* PL: hash of all unbound pools keyed by pool->attrs */ static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); -/* I: attributes used when instantiating standard unbound pools on demand */ -static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; - -/* I: attributes used when instantiating ordered pools on demand */ -static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; - -/* I: attributes of percpu workqueues, which are backed by the static pools */ -static struct workqueue_attrs *percpu_std_wq_attrs[NR_STD_WORKER_POOLS]; - /* * I: kthread_worker to release pwq's. pwq release needs to be bounced to a * process context while holding a pool lock. Bounce to a dedicated kthread @@ -5920,9 +5911,27 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) put_pwq_unlocked(old_pwq); } +/* the attrs @wq asked for, as spelled by its flags */ +static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq) +{ + struct workqueue_attrs *attrs; + + attrs = alloc_workqueue_attrs(); + if (!attrs) + return NULL; + + if (wq->flags & WQ_HIGHPRI) + attrs->nice = HIGHPRI_NICE_LEVEL; + + if (wq->flags & __WQ_ORDERED) + attrs->ordered = true; + + return attrs; +} + static int alloc_and_link_pwqs(struct workqueue_struct *wq) { - bool highpri = wq->flags & WQ_HIGHPRI; + struct workqueue_attrs *attrs; int ret; lockdep_assert_held(&wq_pool_mutex); @@ -5931,23 +5940,24 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) if (!wq->cpu_pwq) goto enomem; - if (!(wq->flags & WQ_UNBOUND)) { - ret = apply_workqueue_attrs_locked(wq, percpu_std_wq_attrs[highpri]); - } else if (wq->flags & __WQ_ORDERED) { - struct pool_workqueue *dfl_pwq; - - ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); - /* there should only be single pwq for ordering guarantee */ - dfl_pwq = rcu_access_pointer(wq->dfl_pwq); - WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node || - wq->pwqs.prev != &dfl_pwq->pwqs_node), - "ordering guarantee broken for workqueue %s\n", wq->name); - } else { - ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); - } + attrs = alloc_wq_std_attrs(wq); + if (!attrs) + goto enomem; + ret = apply_workqueue_attrs_locked(wq, attrs); + free_workqueue_attrs(attrs); if (ret) goto enomem; + + if (wq->flags & __WQ_ORDERED) { + struct pool_workqueue *dfl_pwq = rcu_access_pointer(wq->dfl_pwq); + + /* there should only be single pwq for ordering guarantee */ + WARN(wq->pwqs.next != &dfl_pwq->pwqs_node || + wq->pwqs.prev != &dfl_pwq->pwqs_node, + "ordering guarantee broken for workqueue %s\n", wq->name); + } + return 0; enomem: @@ -8378,28 +8388,6 @@ void __init workqueue_init_early(void) init_cpu_worker_pool(pool, cpu, std_nice[i++]); } - /* create default unbound, ordered and percpu wq attrs */ - for (i = 0; i < NR_STD_WORKER_POOLS; i++) { - struct workqueue_attrs *attrs; - - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - unbound_std_wq_attrs[i] = attrs; - - /* - * An ordered wq should have only one pwq as ordering is - * guaranteed by max_active which is enforced by pwqs. - */ - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - attrs->ordered = true; - ordered_wq_attrs[i] = attrs; - - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - percpu_std_wq_attrs[i] = attrs; - } - system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0); system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0); system_highpri_wq = alloc_workqueue("events_highpri", From 9d4815c14f7faf789aeaa63515024168daf0390c Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 11 Sep 2026 03:26:28 -0700 Subject: [PATCH 15/16] workqueue: BUG_ON() a pwq release before workqueue_init() put_pwq() bounces the release to pwq_release_worker to get out from under pool->lock. workqueue_init() creates that kthread_worker well after workqueue_init_early() has built the system workqueues, so a pwq allocation failing partway through apply_wqattrs_prepare() unwinds through apply_wqattrs_cleanup() and dereferences a NULL worker. There is no way back from that. Running the release inline is not an option, that is what the bounce exists for, and workqueue_init_early() already BUG_ON()s a system workqueue it failed to allocate. BUG_ON() the missing worker instead of oopsing inside kthread_queue_work(). Suggested-by: Tejun Heo Link: https://lore.kernel.org/all/144b3dc986135ebe34f671cdd02bd8aa@kernel.org/ Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 39187f424c34..e618108c6127 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1771,6 +1771,14 @@ static void put_pwq(struct pool_workqueue *pwq) lockdep_assert_held(&pwq->pool->lock); if (likely(--pwq->refcnt)) return; + + /* + * pwq_release_worker is only created in workqueue_init(). Getting + * here before that means an early workqueue allocation failed, which + * is fatal anyway. + */ + BUG_ON(!pwq_release_worker); + /* * @pwq can't be released under pool->lock, bounce to a dedicated * kthread_worker to avoid A-A deadlocks. From bda345b3fc59f2899c50379c64a0734a3e720683 Mon Sep 17 00:00:00 2001 From: Nguyen Ngoc Thang Date: Wed, 14 Oct 2026 22:43:41 +0700 Subject: [PATCH 16/16] workqueue: mark pool->nr_running as __data_racy wq_worker_running() increments pool->nr_running with only preemption disabled, while kick_pool_pick() reads it via need_more_worker() under pool->lock, so the two can run concurrently on different CPUs. KCSAN reports: BUG: KCSAN: data-race in kick_pool_pick / wq_worker_running write to 0xffff888059c2af24 of 4 bytes by task 3007 on cpu 0: wq_worker_running+0x114/0x130 schedule_timeout+0x16e/0x180 io_schedule_timeout+0x4c/0x70 submit_bio_wait+0xd7/0x110 blkdev_issue_flush+0x4c/0x70 ext4_sync_file+0x60d/0x6c0 aio_fsync_work+0x7f/0x270 process_one_work+0x4ae/0xa00 read to 0xffff888059c2af24 of 4 bytes by task 3001 on cpu 1: kick_pool_pick+0x7a/0x3a0 __queue_work+0x72d/0xcf0 queue_work_on+0x9d/0xb0 schedule_on_each_cpu+0x11a/0x2d0 vmstat_refresh+0x23/0x140 proc_sys_call_handler+0x2b9/0x4f0 The race is benign. The increment in wq_worker_running() is the only lockless write to nr_running; decrements and the reset are done under pool->lock, so a reader holding the lock sees either the old or the new count. A stale 0 costs at most a spurious wakeup of an idle worker. A nonzero count is correct because the incrementing worker is running and re-checks the worklist under pool->lock before it stops counting. The increment can also go 1 -> 2, e.g. when a worker sleeping in a work item wakes up after kick_pool() woke another worker. The lockless increment is part of how nr_running is designed to work, as the comment above the field documents. Mark the field __data_racy so that every reader is covered, not just need_more_worker(). Reported-by: syzbot+8922e6b8343f70be92e8@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=8922e6b8343f70be92e8 Signed-off-by: Nguyen Ngoc Thang Signed-off-by: Tejun Heo --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e618108c6127..41754f4126a6 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -212,7 +212,7 @@ struct worker_pool { * but w/ pool->lock held. The readers grab pool->lock and are * guaranteed to see if the counter reached zero. */ - int nr_running; + int __data_racy nr_running; struct list_head worklist; /* L: list of pending works */