From 3dd95f077371f5def65a91c4bb13bd574a62bc3a Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Fri, 7 Aug 2026 03:52:08 +0000 Subject: [PATCH 01/64] sched/core: Don't steal a proxy-exec donor try_steal_cookie() avoids stealing src->core_pick and src->curr before moving a task with the same cookie via move_queued_task_locked(). With proxy-exec, src->donor is the current scheduling context and may differ from src->curr. Stealing it migrates a task that the source rq still treats as current, leaving src's scheduler state for that task stale. For CFS this means cfs_rq->curr points at the stolen entity, and the next pick on the source rq hits the WARN_ON_ONCE in put_prev_entity(). Commit 7de9d4f94638 ("sched: Start blocked_on chain processing in find_proxy_task()") tweaked the fair class logic so that the donor task isn't migrated away while we're running the proxy. Do it similarly for try_steal_cookie() and skip src->donor as well. Fixes: 7de9d4f94638 ("sched: Start blocked_on chain processing in find_proxy_task()") Signed-off-by: Vasily Gorbik Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807035232.1881495-3-jstultz@google.com --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 74724501c623..7adbdccc98c6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6509,7 +6509,7 @@ static bool try_steal_cookie(int this, int that) return false; do { - if (p == src->core_pick || p == src->curr) + if (p == src->core_pick || p == src->curr || p == src->donor) goto next; if (!is_cpu_allowed(p, this)) From 9be817f991e2eba140669d350df2a07d81f26fc8 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 7 Aug 2026 03:52:09 +0000 Subject: [PATCH 02/64] sched/core: Avoid migrating blocked_on tasks Its not useful to migrate blocked_on tasks from other runqueues as the proxy logic will just migrate it back to the owner's rq. So skip blocked_on tasks here. Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807035232.1881495-4-jstultz@google.com --- kernel/sched/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7adbdccc98c6..0f06f8d6647d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6512,6 +6512,9 @@ static bool try_steal_cookie(int this, int that) if (p == src->core_pick || p == src->curr || p == src->donor) goto next; + if (task_is_blocked(p)) + goto next; + if (!is_cpu_allowed(p, this)) goto next; From 09351db90a28ee6a82dc873609edfde02da5415d Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Fri, 7 Aug 2026 03:52:10 +0000 Subject: [PATCH 03/64] sched/core: Don't proxy-exec unmatched cookie lock owners Core scheduling chooses a core-wide cookie before __schedule() installs the next task. With proxy-exec enabled, that task becomes the donor/scheduling context, and find_proxy_task() may then replace the execution context with the runnable mutex owner. If its cookie differs from the selected core cookie, running it would bypass core scheduling's cookie selection. When the final mutex owner found by find_proxy_task() does not match the selected core cookie, stop proxying the donor. If the current execution context is already in the blocked chain, fall back to idle like the existing proxy-exec retry paths do. Otherwise deactivate the donor and let __schedule() pick again. The mutex owner can be picked later under its own cookie. Fixes: 7de9d4f94638 ("sched: Start blocked_on chain processing in find_proxy_task()") [jstultz: Added tweak to ensure we deactivate donor, not runnable owner] Reported-by: K Prateek Nayak Signed-off-by: Vasily Gorbik Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/lkml/10282ce9-f4ae-498f-9b57-f4e1e61fffbc@amd.com/ Link: https://patch.msgid.link/20260807035232.1881495-5-jstultz@google.com --- kernel/sched/core.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0f06f8d6647d..0e6306df3ea9 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7046,6 +7046,14 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) owner->blocked_donor = p; } WARN_ON_ONCE(owner && !owner->on_rq); + + if (owner && !sched_cpu_cookie_match(rq, owner)) { + if (curr_in_chain) + return proxy_resched_idle(rq); + p = donor; /* Deactivate the donor, not the runnable owner */ + clear_task_blocked_on(p, NULL); + goto deactivate; + } return owner; deactivate: From 1f880513859336b1d4f9731d3a67d22d0b3b6aad Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 7 Aug 2026 03:52:11 +0000 Subject: [PATCH 04/64] sched: Switch rq->next_class in proxy_reset_donor() Similar to commit 04f80f8b12a0 ("sched: Switch rq->next_class on proxy_resched_idle()"), we should also set the rq->next_class when we call proxy_reset_donor(). Fixes: f13beb010e4a ("sched: Have try_to_wake_up() handle return-migration for PROXY_WAKING case") Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807035232.1881495-6-jstultz@google.com --- kernel/sched/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0e6306df3ea9..f80b60787260 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3766,6 +3766,7 @@ static inline void proxy_reset_donor(struct rq *rq) WARN_ON_ONCE(rq->donor == rq->curr); put_prev_set_next_task(rq, rq->donor, rq->curr); + rq->next_class = rq->curr->sched_class; rq_set_donor(rq, rq->curr); zap_balance_callbacks(rq); resched_curr(rq); From 6b73a09e943fe84778672362bfbd1cbb3fca9a8a Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 7 Aug 2026 03:52:12 +0000 Subject: [PATCH 05/64] sched: Break out core of attach_tasks() helper into sched.h Pull the core of attach_tasks() out into sched.h so it can be used more generically. Suggested-by: K Prateek Nayak Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807035232.1881495-7-jstultz@google.com --- kernel/sched/fair.c | 16 +--------------- kernel/sched/sched.h | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ade1eceb39b8..b8bd308c2d5b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11194,21 +11194,7 @@ static int detach_tasks(struct lb_env *env) */ static void attach_tasks(struct lb_env *env) { - struct list_head *tasks = &env->tasks; - struct task_struct *p; - struct rq_flags rf; - - rq_lock(env->dst_rq, &rf); - update_rq_clock(env->dst_rq); - - while (!list_empty(tasks)) { - p = list_first_entry(tasks, struct task_struct, se.group_node); - list_del_init(&p->se.group_node); - - attach_task(env->dst_rq, p); - } - - rq_unlock(env->dst_rq, &rf); + __attach_tasks(env->dst_rq, &env->tasks); } #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e656c7059bf8..81f17e62328f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3139,6 +3139,25 @@ static inline void attach_one_task(struct rq *rq, struct task_struct *p) attach_task(rq, p); } +/* + * __attach_tasks() - attaches a list of tasks (using se.group_node) to + * the new rq + */ +static inline void __attach_tasks(struct rq *rq, struct list_head *tasks) +{ + guard(rq_lock)(rq); + update_rq_clock(rq); + + while (!list_empty(tasks)) { + struct task_struct *p; + + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + attach_task(rq, p); + } +} + #ifdef CONFIG_PREEMPT_RT # define SCHED_NR_MIGRATE_BREAK 8 #else From 772d9ffbfd2627779ed9b73155585e63c940bcec Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 7 Aug 2026 03:52:13 +0000 Subject: [PATCH 06/64] sched: Migrate whole chain in proxy_migrate_task() Instead of migrating one task each time through find_proxy_task(), we can walk up the blocked_donor ptrs and migrate the entire current chain in one go. This was broken out of earlier patches and held back while the series was being stabilized, but I wanted to re-introduce it. Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807035232.1881495-8-jstultz@google.com --- kernel/sched/core.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f80b60787260..98410431caef 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6861,9 +6861,9 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, __must_hold(__rq_lockp(rq)) { struct rq *target_rq = cpu_rq(target_cpu); + LIST_HEAD(migrate_list); lockdep_assert_rq_held(rq); - WARN_ON(p == rq->curr); /* * Since we are migrating a blocked donor, it could be rq->donor, * and we want to make sure there aren't any references from this @@ -6876,13 +6876,20 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, * before we release the lock. */ proxy_resched_idle(rq); - - deactivate_task(rq, p, DEQUEUE_NOCLOCK); - proxy_set_task_cpu(p, target_cpu); - + for (; p; p = p->blocked_donor) { + WARN_ON(p == rq->curr); + deactivate_task(rq, p, DEQUEUE_NOCLOCK); + proxy_set_task_cpu(p, target_cpu); + /* + * We can re-use se.group_node to migrate the thing, + * because @p is deactivated (won't be balanced) and + * we hold the rq_lock. + */ + list_add(&p->se.group_node, &migrate_list); + } proxy_release_rq_lock(rq, rf); - attach_one_task(target_rq, p); + __attach_tasks(target_rq, &migrate_list); proxy_reacquire_rq_lock(rq, rf); } From d3d16750693b540299cd105968174556012aba09 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:26 +0100 Subject: [PATCH 07/64] sched: dynamic: Make PREEMPT_DYNAMIC depend on ARCH_HAS_PREEMPT_LAZY On architectures which select ARCH_HAS_PREEMPT_LAZY, it has not been possible to select the NONE and VOLUNTARY preemption models since commit: 7dadeaa6e851 ("sched: Further restrict the preemption modes") ... which was merged in v7.0. All architectures which currently support PREEMPT_DYNAMIC select ARCH_HAS_PREEMPT_LAZY: [mark@lakrids:~/src/linux]% git describe HEAD v7.2-rc1-1-g871a4586ea2e1 [mark@lakrids:~/src/linux]% git grep 'select HAVE_PREEMPT_DYNAMIC_' -- arch arch/arm64/Kconfig: select HAVE_PREEMPT_DYNAMIC_KEY arch/loongarch/Kconfig: select HAVE_PREEMPT_DYNAMIC_KEY arch/powerpc/Kconfig: select HAVE_PREEMPT_DYNAMIC_KEY arch/riscv/Kconfig: select HAVE_PREEMPT_DYNAMIC_KEY arch/s390/Kconfig: select HAVE_PREEMPT_DYNAMIC_KEY arch/x86/Kconfig: select HAVE_PREEMPT_DYNAMIC_CALL [mark@lakrids:~/src/linux]% git grep 'select ARCH_HAS_PREEMPT_LAZY' -- arch arch/arm64/Kconfig: select ARCH_HAS_PREEMPT_LAZY arch/loongarch/Kconfig: select ARCH_HAS_PREEMPT_LAZY arch/powerpc/Kconfig: select ARCH_HAS_PREEMPT_LAZY arch/riscv/Kconfig: select ARCH_HAS_PREEMPT_LAZY arch/s390/Kconfig: select ARCH_HAS_PREEMPT_LAZY arch/x86/Kconfig: select ARCH_HAS_PREEMPT_LAZY ... and hence in practice PREEMPT_DYNAMIC no longer supports the NONE or VOLUNTARY preemption models. Make this official: have PREEMPT_DYNAMIC depend on ARCH_HAS_PREEMPT_LAZY, and remove the trivially unreachable code. Further simplifications will be made in subsequent patches. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-2-mark.rutland@arm.com --- kernel/Kconfig.preempt | 1 + kernel/sched/core.c | 63 ++---------------------------------------- 2 files changed, 4 insertions(+), 60 deletions(-) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index f294dad43bd7..68f321841c2e 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -133,6 +133,7 @@ config PREEMPTION config PREEMPT_DYNAMIC bool "Preemption behaviour defined on boot" depends on HAVE_PREEMPT_DYNAMIC + depends on ARCH_HAS_PREEMPT_LAZY select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY select PREEMPT_BUILD default y if HAVE_PREEMPT_DYNAMIC_CALL diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 98410431caef..84e7685d1930 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7931,20 +7931,10 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); * * * NONE: - * cond_resched <- __cond_resched - * might_resched <- RET0 - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * VOLUNTARY: - * cond_resched <- __cond_resched - * might_resched <- __cond_resched - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * FULL: * cond_resched <- RET0 @@ -7975,21 +7965,11 @@ int preempt_dynamic_mode = preempt_dynamic_undefined; int sched_dynamic_mode(const char *str) { -# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY)) - if (!strcmp(str, "none")) - return preempt_dynamic_none; - - if (!strcmp(str, "voluntary")) - return preempt_dynamic_voluntary; -# endif - if (!strcmp(str, "full")) return preempt_dynamic_full; -# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY if (!strcmp(str, "lazy")) return preempt_dynamic_lazy; -# endif return -EINVAL; } @@ -8011,40 +7991,7 @@ static DEFINE_MUTEX(sched_dynamic_mutex); static void __sched_dynamic_update(int mode) { - /* - * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in - * the ZERO state, which is invalid. - */ - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - switch (mode) { - case preempt_dynamic_none: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: none\n"); - break; - - case preempt_dynamic_voluntary: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: voluntary\n"); - break; - case preempt_dynamic_full: preempt_dynamic_disable(cond_resched); preempt_dynamic_disable(might_resched); @@ -8094,11 +8041,7 @@ __setup("preempt=", setup_preempt_mode); static void __init preempt_dynamic_init(void) { if (preempt_dynamic_mode == preempt_dynamic_undefined) { - if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { - sched_dynamic_update(preempt_dynamic_none); - } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { - sched_dynamic_update(preempt_dynamic_voluntary); - } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { + if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { sched_dynamic_update(preempt_dynamic_lazy); } else { /* Default static call setting, nothing to do */ From 88e0b3bb9930605ad7e5d3ddac7d9875325bebf0 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:27 +0100 Subject: [PATCH 08/64] sched: dynamic: Simplify {cond,might}_resched() PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. In either model, both cond_resched() and might_resched() are always disabled and do nothing. Remove the unnecessary code for these when PREEMPT_DYNAMIC is selected. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-3-mark.rutland@arm.com --- include/linux/kernel.h | 20 ------------------- include/linux/sched.h | 31 +++-------------------------- kernel/sched/core.c | 44 +----------------------------------------- 3 files changed, 4 insertions(+), 91 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24414c79e59a..4b11d1dc0a67 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -43,30 +43,10 @@ struct completion; struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD - extern int __cond_resched(void); # define might_resched() __cond_resched() - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -extern int __cond_resched(void); - -DECLARE_STATIC_CALL(might_resched, __cond_resched); - -static __always_inline void might_resched(void) -{ - static_call_mod(might_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_might_resched(void); -# define might_resched() dynamic_might_resched() - #else - # define might_resched() do { } while (0) - #endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/include/linux/sched.h b/include/linux/sched.h index 8b3d47a325cc..d7cc77181ef9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2134,44 +2134,19 @@ static inline void set_need_resched_current(void) * value indicates whether a reschedule was done in fact. * cond_resched_lock() will drop the spinlock before scheduling, */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) extern int __cond_resched(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -DECLARE_STATIC_CALL(cond_resched, __cond_resched); - -static __always_inline int _cond_resched(void) -{ - return static_call_mod(cond_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_cond_resched(void); - -static __always_inline int _cond_resched(void) -{ - return dynamic_cond_resched(); -} - -#else /* !CONFIG_PREEMPTION */ - static inline int _cond_resched(void) { return __cond_resched(); } - -#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ - -#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ - +#else static inline int _cond_resched(void) { return 0; } - -#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ +#endif #define cond_resched() ({ \ __might_resched(__FILE__, __LINE__, 0); \ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 84e7685d1930..0060efcf5d8d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7794,7 +7794,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) } #endif /* CONFIG_RT_MUTEXES */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) int __sched __cond_resched(void) { if (should_resched(0) && !irqs_disabled()) { @@ -7822,38 +7822,6 @@ int __sched __cond_resched(void) EXPORT_SYMBOL(__cond_resched); #endif -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# define cond_resched_dynamic_enabled __cond_resched -# define cond_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(cond_resched); - -# define might_resched_dynamic_enabled __cond_resched -# define might_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(might_resched); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); -int __sched dynamic_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_cond_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_cond_resched); - -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); -int __sched dynamic_might_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_might_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_might_resched); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /* * __cond_resched_lock() - if a reschedule is pending, drop the given lock, * call schedule, and on return reacquire the lock. @@ -7923,8 +7891,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:cond_resched - * SC:might_resched * SC:preempt_schedule * SC:preempt_schedule_notrace * SC:irqentry_exit_cond_resched @@ -7937,16 +7903,12 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); * (unselectable) * * FULL: - * cond_resched <- RET0 - * might_resched <- RET0 * preempt_schedule <- preempt_schedule * preempt_schedule_notrace <- preempt_schedule_notrace * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * cond_resched <- RET0 - * might_resched <- RET0 * preempt_schedule <- preempt_schedule * preempt_schedule_notrace <- preempt_schedule_notrace * irqentry_exit_cond_resched <- irqentry_exit_cond_resched @@ -7993,8 +7955,6 @@ static void __sched_dynamic_update(int mode) { switch (mode) { case preempt_dynamic_full: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); preempt_dynamic_enable(preempt_schedule); preempt_dynamic_enable(preempt_schedule_notrace); preempt_dynamic_enable(irqentry_exit_cond_resched); @@ -8004,8 +7964,6 @@ static void __sched_dynamic_update(int mode) break; case preempt_dynamic_lazy: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); preempt_dynamic_enable(preempt_schedule); preempt_dynamic_enable(preempt_schedule_notrace); preempt_dynamic_enable(irqentry_exit_cond_resched); From b9d267b9d632feec9ddd1ce12078b7113f429a51 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:28 +0100 Subject: [PATCH 09/64] sched: dynamic: Simplify preempt_schedule{,_notrace}() PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. In either model, both preempt_schedule() and preempt_schedule_notrace() are always called and never disabled. Remove the unnecessary code for these when PREEMPT_DYNAMIC is selected. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-4-mark.rutland@arm.com --- arch/arm64/include/asm/preempt.h | 10 ------ arch/s390/include/asm/preempt.h | 11 ------- arch/x86/include/asm/preempt.h | 28 ----------------- include/asm-generic/preempt.h | 10 ------ kernel/sched/core.c | 52 -------------------------------- 5 files changed, 111 deletions(-) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index 9ecc2766a9f2..e86e850e9b17 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -104,19 +104,9 @@ static inline bool should_resched(int preempt_offset) void preempt_schedule(void); void preempt_schedule_notrace(void); -#ifdef CONFIG_PREEMPT_DYNAMIC - -void dynamic_preempt_schedule(void); -#define __preempt_schedule() dynamic_preempt_schedule() -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* CONFIG_PREEMPT_DYNAMIC */ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC */ #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 5560d5fca2a3..60c6f019ec45 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -155,20 +155,9 @@ static __always_inline int __preempt_count_sub_return(int val) void preempt_schedule(void); void preempt_schedule_notrace(void); -#ifdef CONFIG_PREEMPT_DYNAMIC - -void dynamic_preempt_schedule(void); -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule() dynamic_preempt_schedule() -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* CONFIG_PREEMPT_DYNAMIC */ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC */ - #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index fafb6f8cdac3..d16d9c2f7b08 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -137,43 +137,15 @@ static __always_inline bool should_resched(int preempt_offset) extern asmlinkage void preempt_schedule(void); extern asmlinkage void preempt_schedule_thunk(void); -#define preempt_schedule_dynamic_enabled preempt_schedule_thunk -#define preempt_schedule_dynamic_disabled NULL - extern asmlinkage void preempt_schedule_notrace(void); extern asmlinkage void preempt_schedule_notrace_thunk(void); -#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk -#define preempt_schedule_notrace_dynamic_disabled NULL - -#ifdef CONFIG_PREEMPT_DYNAMIC - -DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); - -#define __preempt_schedule() \ -do { \ - __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \ - asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \ -} while (0) - -DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); - -#define __preempt_schedule_notrace() \ -do { \ - __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \ - asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \ -} while (0) - -#else /* PREEMPT_DYNAMIC */ - #define __preempt_schedule() \ asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT); #define __preempt_schedule_notrace() \ asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT); -#endif /* PREEMPT_DYNAMIC */ - #endif /* PREEMPTION */ #undef __pc_op diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c8683c046615..1adddeab8545 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -96,19 +96,9 @@ static __always_inline bool should_resched(int preempt_offset) extern asmlinkage void preempt_schedule(void); extern asmlinkage void preempt_schedule_notrace(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -void dynamic_preempt_schedule(void); -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule() dynamic_preempt_schedule() -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0060efcf5d8d..844f37465d62 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7500,27 +7500,6 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) NOKPROBE_SYMBOL(preempt_schedule); EXPORT_SYMBOL(preempt_schedule); -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# ifndef preempt_schedule_dynamic_enabled -# define preempt_schedule_dynamic_enabled preempt_schedule -# define preempt_schedule_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); -void __sched notrace dynamic_preempt_schedule(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) - return; - preempt_schedule(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule); -EXPORT_SYMBOL(dynamic_preempt_schedule); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /** * preempt_schedule_notrace - preempt_schedule called by tracing * @@ -7573,27 +7552,6 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) } EXPORT_SYMBOL_GPL(preempt_schedule_notrace); -#ifdef CONFIG_PREEMPT_DYNAMIC -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# ifndef preempt_schedule_notrace_dynamic_enabled -# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace -# define preempt_schedule_notrace_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); -void __sched notrace dynamic_preempt_schedule_notrace(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) - return; - preempt_schedule_notrace(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); -EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); -# endif -#endif - #endif /* CONFIG_PREEMPTION */ /* @@ -7891,8 +7849,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:preempt_schedule - * SC:preempt_schedule_notrace * SC:irqentry_exit_cond_resched * * @@ -7903,14 +7859,10 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); * (unselectable) * * FULL: - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- true */ @@ -7955,8 +7907,6 @@ static void __sched_dynamic_update(int mode) { switch (mode) { case preempt_dynamic_full: - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_disable(preempt_lazy); if (mode != preempt_dynamic_mode) @@ -7964,8 +7914,6 @@ static void __sched_dynamic_update(int mode) break; case preempt_dynamic_lazy: - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_enable(preempt_lazy); if (mode != preempt_dynamic_mode) From aa4178f63847f86e3b27b9e53ee8ccbd6ed6bf8c Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:29 +0100 Subject: [PATCH 10/64] sched: dynamic: Simplify irqentry_exit_cond_resched() PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. In either model, irqentry_exit_cond_resched() is always called and never disabled. Remove the unnecessary code for this when PREEMPT_DYNAMIC is selected. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-5-mark.rutland@arm.com --- include/linux/irq-entry-common.h | 17 +---------------- kernel/entry/common.c | 17 ++--------------- kernel/sched/core.c | 7 ------- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..2be273bb68f0 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -346,22 +346,7 @@ typedef struct irqentry_state { * * Conditional reschedule with additional sanity checks. */ -void raw_irqentry_exit_cond_resched(void); - -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched -#define irqentry_exit_cond_resched_dynamic_disabled NULL -DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void); -#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() -#endif -#else /* CONFIG_PREEMPT_DYNAMIC */ -#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() -#endif /* CONFIG_PREEMPT_DYNAMIC */ +void irqentry_exit_cond_resched(void); /** * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler diff --git a/kernel/entry/common.c b/kernel/entry/common.c index e3d381fd3d25..e234b04373fe 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -123,7 +123,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) /** * arch_irqentry_exit_need_resched - Architecture specific need resched function * - * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed. + * Invoked from irqentry_exit_cond_resched() to check if resched is needed. * Defaults return true. * * The main purpose is to permit arch to avoid preemption of a task from an IRQ. @@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void); static inline bool arch_irqentry_exit_need_resched(void) { return true; } #endif -void raw_irqentry_exit_cond_resched(void) +void irqentry_exit_cond_resched(void) { if (!preempt_count()) { /* Sanity check RCU and thread stack */ @@ -145,19 +145,6 @@ void raw_irqentry_exit_cond_resched(void) preempt_schedule_irq(); } } -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) - return; - raw_irqentry_exit_cond_resched(); -} -#endif -#endif noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 844f37465d62..0e9022f2d05d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7849,9 +7849,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:irqentry_exit_cond_resched - * - * * NONE: * (unselectable) * @@ -7859,11 +7856,9 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); * (unselectable) * * FULL: - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- true */ @@ -7907,14 +7902,12 @@ static void __sched_dynamic_update(int mode) { switch (mode) { case preempt_dynamic_full: - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_disable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: full\n"); break; case preempt_dynamic_lazy: - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_enable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: lazy\n"); From 5b9a28eeed37513778ca6b636253aa8dcf39d55f Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:30 +0100 Subject: [PATCH 11/64] sched: dynamic: Remove HAVE_PREEMPT_DYNAMIC_{CALL,KEY} PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. Switching model only changes the behaviour of dynamic_preempt_lazy(), which uses a static key. There are no other static calls or static keys, and so there's no need for HAVE_PREEMPT_DYNAMIC_CALL or HAVE_PREEMPT_DYNAMIC_KEY. The static key used by dynamic_preempt_lazy() is entirely local to kernel/sched/core.c, and any architecture which selects ARCH_HAS_PREEMPT_LAZY implements the necessary support. Remove PREEMPT_DYNAMIC's dependencies on HAVE_PREEMPT_DYNAMIC_CALL and HAVE_PREEMPT_DYNAMIC_KEY entirely, leaving the dependency on ARCH_HAS_PREEMPT_LAZY. For architectures which previously selected HAVE_PREEMPT_DYNAMIC_KEY, PREEMPT_DYNAMIC will now be selected by default, matching x86. As the runtime impact is limited to dynamic_preempt_lazy(), this shouldn't be as concerning as previously (e.g. where calls to {cond,might}_resched() stubs could introduce a measurable penalty). As all of this can work without jump labels, JUMP_LABEL is not selected explicitly. It is obviously preferable to have JUMP_LABEL enabled, but this is not functionally necessary. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Jinjie Ruan Reviewed-by: Shrikanth Hegde Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-6-mark.rutland@arm.com --- arch/Kconfig | 38 -------------------------------------- arch/arm64/Kconfig | 1 - arch/loongarch/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/riscv/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/x86/Kconfig | 1 - kernel/Kconfig.preempt | 8 ++------ kernel/sched/core.c | 10 ---------- 9 files changed, 2 insertions(+), 60 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 45c657772362..c0a6a13b06a1 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1712,44 +1712,6 @@ config HAVE_STATIC_CALL_INLINE depends on HAVE_STATIC_CALL select OBJTOOL -config HAVE_PREEMPT_DYNAMIC - bool - -config HAVE_PREEMPT_DYNAMIC_CALL - bool - depends on HAVE_STATIC_CALL - select HAVE_PREEMPT_DYNAMIC - help - An architecture should select this if it can handle the preemption - model being selected at boot time using static calls. - - Where an architecture selects HAVE_STATIC_CALL_INLINE, any call to a - preemption function will be patched directly. - - Where an architecture does not select HAVE_STATIC_CALL_INLINE, any - call to a preemption function will go through a trampoline, and the - trampoline will be patched. - - It is strongly advised to support inline static call to avoid any - overhead. - -config HAVE_PREEMPT_DYNAMIC_KEY - bool - depends on HAVE_ARCH_JUMP_LABEL - select HAVE_PREEMPT_DYNAMIC - help - An architecture should select this if it can handle the preemption - model being selected at boot time using static keys. - - Each preemption function will be given an early return based on a - static key. This should have slightly lower overhead than non-inline - static calls, as this effectively inlines each trampoline into the - start of its callee. This may avoid redundant work, and may - integrate better with CFI schemes. - - This will have greater overhead than using inline static calls as - the call to the preemption function cannot be entirely elided. - config ARCH_WANT_LD_ORPHAN_WARN bool help diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5a51b0ef944..876ef299e169 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -217,7 +217,6 @@ config ARM64 select HAVE_PERF_EVENTS_NMI if ARM64_PSEUDO_NMI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_POSIX_CPU_TIMERS_TASK_WORK diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index a21f51e5815e..9480ccf73b26 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -170,7 +170,6 @@ config LOONGARCH select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC select HAVE_RETHOOK diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 2580e27e4328..143381d267ce 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -279,7 +279,6 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_RETHOOK if KPROBES select HAVE_REGS_AND_STACK_ACCESS_API diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f8e26c4bed2b..66096a050f88 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -195,7 +195,6 @@ config RISCV select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RETHOOK select HAVE_RSEQ diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4b51bc6e8948..2682a2bacc58 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -246,7 +246,6 @@ config S390 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RETHOOK diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 15fd9ec5ecac..5a84998b6021 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -297,7 +297,6 @@ config X86 select HAVE_STACK_VALIDATION if HAVE_OBJTOOL select HAVE_STATIC_CALL select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL - select HAVE_PREEMPT_DYNAMIC_CALL select HAVE_RSEQ select HAVE_RUST if X86_64 select HAVE_SYSCALL_TRACEPOINTS diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 68f321841c2e..985aea617cfe 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -132,11 +132,9 @@ config PREEMPTION config PREEMPT_DYNAMIC bool "Preemption behaviour defined on boot" - depends on HAVE_PREEMPT_DYNAMIC depends on ARCH_HAS_PREEMPT_LAZY - select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY select PREEMPT_BUILD - default y if HAVE_PREEMPT_DYNAMIC_CALL + default y help This option allows to define the preemption model on the kernel command line parameter and thus override the default preemption @@ -146,9 +144,7 @@ config PREEMPT_DYNAMIC provide a pre-built kernel binary to reduce the number of kernel flavors they offer while still offering different usecases. - The runtime overhead is negligible with HAVE_STATIC_CALL_INLINE enabled - but if runtime patching is not available for the specific architecture - then the potential overhead should be considered. + The runtime overhead is negligible. Interesting if you want the same pre-built kernel should be used for both Server and Desktop workloads. diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0e9022f2d05d..05923200ef40 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7886,16 +7886,6 @@ int sched_dynamic_mode(const char *str) # define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) # define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) -# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) -# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) -# else -# error "Unsupported PREEMPT_DYNAMIC mechanism" -# endif - static DEFINE_MUTEX(sched_dynamic_mutex); static void __sched_dynamic_update(int mode) From 9650ce11f2e377310332d87a902dc4df3f59b905 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:31 +0100 Subject: [PATCH 12/64] sched: dynamic: Simplify preempt model accessors PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. When CONFIG_PREEMPT_DYNAMIC=y, the NONE and VOLUNTARY models cannot be used, and neither CONFIG_PREEMPT_NONE nor CONFIG_PREEMPT_VOLUNTARY can be selected. Simplify the preempt model accessors accordingly: remove the out-of-line implementations of preempt_model_none() and preempt_model_voluntary(), and remove the preempt_dynamic_{none,voluntary} values. There are no current users of preempt_model_voluntary(), but I've kept it around for consistency, with every scheduler model having its own preempt_model_*() accessor. Suggested-by: Shrikanth Hegde Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Jinjie Ruan Reviewed-by: Shrikanth Hegde Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-7-mark.rutland@arm.com --- include/linux/preempt.h | 20 ++++++++++---------- kernel/sched/core.c | 4 ---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 8299657f0f86..40310675a7cc 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -502,21 +502,11 @@ DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_n #ifdef CONFIG_PREEMPT_DYNAMIC -extern bool preempt_model_none(void); -extern bool preempt_model_voluntary(void); extern bool preempt_model_full(void); extern bool preempt_model_lazy(void); #else -static inline bool preempt_model_none(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_NONE); -} -static inline bool preempt_model_voluntary(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); -} static inline bool preempt_model_full(void) { return IS_ENABLED(CONFIG_PREEMPT); @@ -529,6 +519,16 @@ static inline bool preempt_model_lazy(void) #endif +static inline bool preempt_model_none(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_NONE); +} + +static inline bool preempt_model_voluntary(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); +} + static inline bool preempt_model_rt(void) { return IS_ENABLED(CONFIG_PREEMPT_RT); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 05923200ef40..c9dd6f4d42a4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7864,8 +7864,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); enum { preempt_dynamic_undefined = -1, - preempt_dynamic_none, - preempt_dynamic_voluntary, preempt_dynamic_full, preempt_dynamic_lazy, }; @@ -7950,8 +7948,6 @@ static void __init preempt_dynamic_init(void) } \ EXPORT_SYMBOL_GPL(preempt_model_##mode) -PREEMPT_MODEL_ACCESSOR(none); -PREEMPT_MODEL_ACCESSOR(voluntary); PREEMPT_MODEL_ACCESSOR(full); PREEMPT_MODEL_ACCESSOR(lazy); From a5576ebce9201e2abc820348a494e666de8efe64 Mon Sep 17 00:00:00 2001 From: Hongyan Xia Date: Wed, 19 Aug 2026 08:12:09 +0000 Subject: [PATCH 13/64] sched: Convert paravirt_steal to new static key APIs paravirt_steal_rq_enabled and paravirt_steal_enabled use raw static_key APIs which are now deprecated. Use the new API instead. No functional change. Signed-off-by: Hongyan Xia Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/20260819081207.12150-1-hongyan.xia@transsion.com --- arch/arm64/kernel/paravirt.c | 4 ++-- arch/loongarch/kernel/paravirt.c | 4 ++-- arch/powerpc/platforms/pseries/setup.c | 4 ++-- arch/riscv/kernel/paravirt.c | 4 ++-- arch/x86/kernel/cpu/vmware.c | 4 ++-- arch/x86/kernel/kvm.c | 4 ++-- drivers/xen/time.c | 4 ++-- include/linux/sched/cputime.h | 6 +++--- kernel/sched/core.c | 4 ++-- kernel/sched/cputime.c | 4 ++-- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index 572efb96b23f..30bf61d031eb 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -157,9 +157,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, para_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("using stolen time PV\n"); diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index 10821cce554c..e8965a3f8082 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -308,10 +308,10 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, paravt_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif if (static_key_enabled(&virt_preempt_key)) diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index f29e7547c995..c181df4dec70 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -876,9 +876,9 @@ static void __init pSeries_setup_arch(void) static_branch_enable(&shared_processor); pv_spinlocks_init(); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif } diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 5f56be79cd06..9c13a6f1ea2a 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -116,9 +116,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, pv_time_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("Computing paravirt steal-time\n"); diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 34b73573b108..f7ab9e7902cf 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -328,9 +328,9 @@ static int vmware_cpu_down_prepare(unsigned int cpu) static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6b0a5861ccb8..2ce01fdb8b4b 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1054,9 +1054,9 @@ const __initconst struct hypervisor_x86 x86_hyper_kvm = { static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/drivers/xen/time.c b/drivers/xen/time.c index a2be0a4d45b0..a02d48a2aa68 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -169,7 +169,7 @@ void __init xen_time_setup_guest(void) static_call_update(pv_steal_clock, xen_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (xen_runstate_remote) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index e90efaf6d26e..694126411dfe 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -182,9 +182,9 @@ extern unsigned long long task_sched_runtime(struct task_struct *task); #ifdef CONFIG_PARAVIRT -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; +#include +DECLARE_STATIC_KEY_FALSE(paravirt_steal_enabled); +DECLARE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN u64 dummy_steal_clock(int cpu); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c9dd6f4d42a4..4e9776681d0e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -805,7 +805,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_rq_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #endif static void update_rq_clock_task(struct rq *rq, s64 delta) @@ -844,7 +844,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) } #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - if (static_key_false((¶virt_steal_rq_enabled))) { + if (static_branch_unlikely(¶virt_steal_rq_enabled)) { u64 prev_steal; steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 06bddaa738e5..f16970ca81d0 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -255,7 +255,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * occasion account more time than the calling functions think elapsed. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN static u64 native_steal_clock(int cpu) @@ -270,7 +270,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT - if (static_key_false(¶virt_steal_enabled)) { + if (static_branch_unlikely(¶virt_steal_enabled)) { u64 steal; steal = paravirt_steal_clock(smp_processor_id()); From 2a672daa4b272d092e193eb40376b4a8063e25fa Mon Sep 17 00:00:00 2001 From: Hongyan Xia Date: Wed, 19 Aug 2026 08:13:14 +0000 Subject: [PATCH 14/64] sched/feat: Use the new static key API for sched_feat Both raw static keys and static_key_true/false() have been deprecated. Migrate to new API. It's a bit tricky to convert the sched_feat array to the new API because the new one has two different types for true and false keys. Use a union to wrap it. No functional change. Signed-off-by: Hongyan Xia Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260819081312.12447-1-hongyan.xia@transsion.com --- kernel/sched/debug.c | 10 +++++----- kernel/sched/sched.h | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 72236db67983..9883e4c4976b 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -73,13 +73,13 @@ static int sched_feat_show(struct seq_file *m, void *v) #ifdef CONFIG_JUMP_LABEL -#define jump_label_key__true STATIC_KEY_INIT_TRUE -#define jump_label_key__false STATIC_KEY_INIT_FALSE +#define jump_label_key__true { .key_true = STATIC_KEY_TRUE_INIT } +#define jump_label_key__false { .key_false = STATIC_KEY_FALSE_INIT } #define SCHED_FEAT(name, enabled) \ jump_label_key__##enabled , -struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { +union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR] = { #include "features.h" }; @@ -87,12 +87,12 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { static void sched_feat_disable(int i) { - static_key_disable_cpuslocked(&sched_feat_keys[i]); + static_branch_disable_cpuslocked(&sched_feat_keys[i].key_true); } static void sched_feat_enable(int i) { - static_key_enable_cpuslocked(&sched_feat_keys[i]); + static_branch_enable_cpuslocked(&sched_feat_keys[i].key_false); } #else /* !CONFIG_JUMP_LABEL: */ static void sched_feat_disable(int i) { }; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 81f17e62328f..6c3ad70e58b8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2447,16 +2447,25 @@ extern __read_mostly unsigned int sysctl_sched_features; #ifdef CONFIG_JUMP_LABEL -#define SCHED_FEAT(name, enabled) \ -static __always_inline bool static_branch_##name(struct static_key *key) \ -{ \ - return static_key_##enabled(key); \ +union sched_feat_key { + struct static_key_true key_true; + struct static_key_false key_false; +}; + +#define sched_feat_branch_true(key) static_branch_likely(&(key)->key_true) +#define sched_feat_branch_false(key) static_branch_unlikely(&(key)->key_false) + +#define SCHED_FEAT(name, enabled) \ +static __always_inline bool \ +static_branch_##name(union sched_feat_key *key) \ +{ \ + return sched_feat_branch_##enabled(key); \ } #include "features.h" #undef SCHED_FEAT -extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; +extern union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR]; #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) #else /* !CONFIG_JUMP_LABEL: */ From f549101187c875c2016072cadcb909d228532ae8 Mon Sep 17 00:00:00 2001 From: Liang Hao Date: Sun, 16 Aug 2026 11:26:43 +0800 Subject: [PATCH 15/64] sched/deadline: check start_dl_timer expiry with ktime_before() Replace ktime_us_delta() < 0 with ktime_before() so the past-expiry test uses the same resolution as act/now. Signed-off-by: Liang Hao Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juri Lelli Link: https://patch.msgid.link/20260816032643.44969-1-haohlliang@gmail.com --- kernel/sched/deadline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0663c00c41c0..de6a361a87c7 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se) * chosen as the deadline is too small, don't even try to * start the timer in the past! */ - if (ktime_us_delta(act, now) < 0) + if (ktime_before(act, now)) return 0; /* From 879eaa76e6086ab4aff77a669bfb50b7db03e0b4 Mon Sep 17 00:00:00 2001 From: Vladimir Zapolskiy Date: Wed, 19 Aug 2026 11:01:58 +0300 Subject: [PATCH 16/64] sched: Remove unneeded function type cast in do_balance_callbacks() It's a trivial and non-functional change, the removed explicit function type cast does not convert the function type of struct balance_callback member (*func), and the cast can be removed. Signed-off-by: Vladimir Zapolskiy Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260819080159.105433-1-vz@kernel.org --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4e9776681d0e..545f49491bcd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5143,7 +5143,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) lockdep_assert_rq_held(rq); while (head) { - func = (void (*)(struct rq *))head->func; + func = head->func; next = head->next; head->next = NULL; head = next; From ef9293b3b797228fead10b55ed6bfb99bb7976b4 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 2 Sep 2026 11:16:37 +0100 Subject: [PATCH 17/64] sched: dynamic: Fix preemption model strings We recently removed the enum values 'preempt_dynamic_none' and 'preempt_dynamic_voluntary', but forgot to update preempt_modes[] accordingly. Due to this, preempt_model_str() and sched_dynamic_show() produce incorrect output, with the wrong preemption model string in backtraces, and empty output for /sys/kernel/debug/sched/preempt. Update preempt_modes[] to account for the removed enum values. Given that CONFIG_PREEMPT_DYNAMIC now depends on CONFIG_ARCH_HAS_PREEMPT_LAZY, and given that only 'preempt_dynamic_full' and 'preempt_dynamic_lazy' remain, sched_dynamic_show() no longer needs to skip any model strings. Fixes: 9650ce11f2e3 ("sched: dynamic: Simplify preempt model accessors") Reported-by: Jinjie Ruan Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260902101637.232129-1-mark.rutland@arm.com --- kernel/sched/core.c | 2 +- kernel/sched/debug.c | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 545f49491bcd..5a61650f12b7 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7960,7 +7960,7 @@ static inline void preempt_dynamic_init(void) { } #endif /* CONFIG_PREEMPT_DYNAMIC */ const char *preempt_modes[] = { - "none", "voluntary", "full", "lazy", NULL, + "full", "lazy", NULL, }; const char *preempt_model_str(void) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 9883e4c4976b..fda10b3fa249 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -280,16 +280,10 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, static int sched_dynamic_show(struct seq_file *m, void *v) { - int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2; int mode = READ_ONCE(preempt_dynamic_mode); - int j; - /* Count entries in NULL terminated preempt_modes */ - for (j = 0; preempt_modes[j]; j++) - ; - j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY); - - for (; i < j; i++) { + /* Stop at NULL terminator */ + for (int i = 0; preempt_modes[i]; i++) { if (mode == i) seq_puts(m, "("); seq_puts(m, preempt_modes[i]); From e81ee06308379a5f2ededf997bcf17551bce5db7 Mon Sep 17 00:00:00 2001 From: Eric Kim Date: Tue, 18 Aug 2026 12:48:29 +0900 Subject: [PATCH 18/64] sched/fair: Reset NUMA fault locality after scan period update When updating the task scan period for NUMA locality checks, update_task_scan_period() checks whether there were no faults or failed migration on the last scan window, at which it prolongs the scan period. However, p->numa_faults_locality, which is used to check for migration failure and number of faults at the previous scan window is not cleared after changing the scan period, which unintentionally increases numa_scan_period up to numa_scan_period_max even when there were no migration failures or no faults at previous scan. Fix this by jumping to the out label on early exit at this case to ensure p->numa_faults_locality is always cleared before returning. Closes: https://lore.kernel.org/all/20250404095354.311156-1-qlsdnjs236@chungbuk.ac.kr/ Reported-by: Binwon Song Signed-off-by: Eric Kim Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://patch.msgid.link/Message-ID: --- kernel/sched/fair.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b8bd308c2d5b..a42360ca94c3 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3514,7 +3514,7 @@ static void update_task_scan_period(struct task_struct *p, p->mm->numa_next_scan = jiffies + msecs_to_jiffies(p->numa_scan_period); - return; + goto out; } /* @@ -3558,7 +3558,10 @@ static void update_task_scan_period(struct task_struct *p, p->numa_scan_period = clamp(p->numa_scan_period + diff, task_scan_min(p), task_scan_max(p)); - memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); + +out: + memset(p->numa_faults_locality, 0, + sizeof(p->numa_faults_locality)); } /* From a9b3c7570564de40acb0998ab85c89d546b2122f Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Tue, 22 Sep 2026 08:46:41 +0200 Subject: [PATCH 19/64] sched/headers: Replace __ASSEMBLY__ with __ASSEMBLER__ in the header While the GCC and Clang compilers already define __ASSEMBLER__ auto- matically when compiling assembly code, __ASSEMBLY__ is a macro that only gets defined by the Makefiles in the kernel. This can be very confusing when switching between userspace and kernelspace coding, or when dealing with uapi headers that rather should use __ASSEMBLER__ instead. Let's standardize now on the __ASSEMBLER__ macro to avoid this confusion. Signed-off-by: Thomas Huth Signed-off-by: Ingo Molnar Cc: Peter Zijlstra Link: https://patch.msgid.link/20260922064641.61120-1-thuth@redhat.com --- include/uapi/linux/sched.h | 2 +- tools/perf/trace/beauty/include/uapi/linux/sched.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 33a4624285cd..19ffeba89428 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -53,7 +53,7 @@ */ #define UNSHARE_EMPTY_MNTNS 0x00100000 /* Unshare an empty mount namespace. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /** * struct clone_args - arguments for the clone3 syscall * @flags: Flags for the new process as listed above. diff --git a/tools/perf/trace/beauty/include/uapi/linux/sched.h b/tools/perf/trace/beauty/include/uapi/linux/sched.h index 33a4624285cd..19ffeba89428 100644 --- a/tools/perf/trace/beauty/include/uapi/linux/sched.h +++ b/tools/perf/trace/beauty/include/uapi/linux/sched.h @@ -53,7 +53,7 @@ */ #define UNSHARE_EMPTY_MNTNS 0x00100000 /* Unshare an empty mount namespace. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /** * struct clone_args - arguments for the clone3 syscall * @flags: Flags for the new process as listed above. From c72945693b90423f1b46ac2dbc8749c2b7804fc8 Mon Sep 17 00:00:00 2001 From: "Shubhang Kaushik (Ampere)" Date: Thu, 17 Sep 2026 16:06:59 -0700 Subject: [PATCH 20/64] sched: Restart fair hrtick after same-task repicks Fair hrticks are one-shot timers. A same-task repick after hrtick expiry skips set_next_task_fair() and leaves no hrtick armed for the next fair preemption point: hrtick() -> task_tick_fair(..., queued=1) -> entity_tick() -> resched_curr() -> schedule() -> pick_task_fair() selects current -> put_prev_set_next_task(rq, prev, next) -> next == prev -> return Introduce SNT_REPICK for the next == prev path: put_prev_set_next_task(rq, prev, next) -> next == prev -> next->sched_class->set_next_task(rq, next, SNT_REPICK) -> set_next_task_fair() -> hrtick_start_fair() set_next_task_fair() skips task-transition work for SNT_REPICK and restarts the fair hrtick. hrtick_start() records the delay during schedule(), and hrtick_schedule_exit() rearms the timer. Do not restart the SCHED_DEADLINE hrtick for SNT_REPICK. Same-task repicks skip put_prev_task() for both fair and SCHED_DEADLINE, but pick_task_fair() refreshes the selected fair entity before selecting it again. hrtick_start_fair() reads that refreshed entity, including with group scheduling. The SCHED_DEADLINE picker has no corresponding current-runtime update. Restarting its hrtick on SNT_REPICK could therefore use stale dl_se->runtime and delay the next runtime enforcement point. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Shubhang Kaushik (Ampere) Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Zhan Xusheng Link: https://patch.msgid.link/20260917-sched-fair-hrtick-restart-v4-1-4dd1414da81a@gentwo.org --- kernel/sched/core.c | 2 +- kernel/sched/deadline.c | 7 +++++-- kernel/sched/ext/ext.c | 5 ++++- kernel/sched/fair.c | 21 ++++++++++++++++----- kernel/sched/idle.c | 5 ++++- kernel/sched/rt.c | 7 +++++-- kernel/sched/sched.h | 16 ++++++++++++---- kernel/sched/stop_task.c | 5 ++++- 8 files changed, 51 insertions(+), 17 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 5a61650f12b7..d368aaf68c8e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7240,7 +7240,7 @@ static void __sched notrace __schedule(int sched_mode) * on_cpu. */ donor->sched_class->put_prev_task(rq, donor, donor); - donor->sched_class->set_next_task(rq, donor, true); + donor->sched_class->set_next_task(rq, donor, SNT_PICK); } } else { rq_set_donor(rq, next); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index de6a361a87c7..c0ebdcde5fe5 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2773,11 +2773,14 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se) * DL keeps current in tree, because ->deadline is not typically changed while * a task is runnable. */ -static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_dl(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_dl_entity *dl_se = &p->dl; struct dl_rq *dl_rq = &rq->dl; + if (type == SNT_REPICK) + return; + p->se.exec_start = rq_clock_task(rq); if (on_dl_rq(&p->dl)) update_stats_wait_end_dl(dl_rq, dl_se); @@ -2788,7 +2791,7 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) WARN_ON_ONCE(dl_rq->curr); dl_rq->curr = dl_se; - if (!first) + if (type != SNT_PICK) return; if (rq->donor->sched_class != &dl_sched_class) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 10af28a9f2c0..53275bc7c029 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3001,10 +3001,13 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev return verdict; } -static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) { struct scx_sched *sch = scx_task_sched(p); + if (type == SNT_REPICK) + return; + if (p->scx.flags & SCX_TASK_QUEUED) { /* * Core-sched might decide to execute @p before it is diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a42360ca94c3..36861f435d8a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -15234,14 +15234,18 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_fair(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_entity *se = &p->se; - bool throttled = false; struct cfs_rq *cfs_rq = &rq->cfs; unsigned long weight = NICE_0_LOAD; + bool first = type == SNT_PICK; + bool throttled = false; bool on_rq = se->on_rq; + if (type == SNT_REPICK) + goto repick; + clear_buddies(cfs_rq, se); if (on_rq) @@ -15285,11 +15289,18 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) WARN_ON_ONCE(se->sched_delayed); - if (hrtick_enabled_fair(rq)) - hrtick_start_fair(rq, p); - update_misfit_status(p, rq); sched_fair_update_stop_tick(rq, p); + +repick: + /* + * A same-task repick skips put_prev_task_fair(), but + * pick_task_fair() refreshed the entity hrtick_start_fair() reads + * before selecting it again. rq->cfs.curr identifies that entity, + * including with group scheduling. + */ + if (hrtick_enabled_fair(rq)) + hrtick_start_fair(rq, p); } void init_cfs_rq(struct cfs_rq *cfs_rq) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index eb73b65ce6c4..76f3c84ca684 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -487,8 +487,11 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct t update_rq_avg_idle(rq); } -static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) +static void set_next_task_idle(struct rq *rq, struct task_struct *next, enum snt_e type) { + if (type == SNT_REPICK) + return; + update_idle_core(rq); scx_update_idle(rq, true, true); schedstat_inc(rq->sched_goidle); diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 85303add726d..1535046a23ff 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1654,11 +1654,14 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) check_preempt_equal_prio(rq, p); } -static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first) +static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_rt_entity *rt_se = &p->rt; struct rt_rq *rt_rq = &rq->rt; + if (type == SNT_REPICK) + return; + p->se.exec_start = rq_clock_task(rq); if (on_rt_rq(&p->rt)) update_stats_wait_end_rt(rt_rq, rt_se); @@ -1666,7 +1669,7 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f /* The running task is never eligible for pushing */ dequeue_pushable_task(rq, p); - if (!first) + if (type != SNT_PICK) return; /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6c3ad70e58b8..1b64be5771ca 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2630,6 +2630,12 @@ struct affinity_context { extern s64 update_curr_common(struct rq *rq); +enum snt_e { + SNT_NORMAL, /* set_next_task() */ + SNT_PICK, /* put_prev_set_next_task(): prev != next */ + SNT_REPICK, /* put_prev_set_next_task(): prev == next */ +}; + struct sched_class { #ifdef CONFIG_UCLAMP_TASK @@ -2687,7 +2693,7 @@ struct sched_class { * __schedule: rq->lock */ void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_struct *next); - void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); + void (*set_next_task)(struct rq *rq, struct task_struct *p, enum snt_e type); /* * select_task_rq: p->pi_lock @@ -2790,7 +2796,7 @@ static inline void put_prev_task(struct rq *rq, struct task_struct *prev) static inline void set_next_task(struct rq *rq, struct task_struct *next) { - next->sched_class->set_next_task(rq, next, false); + next->sched_class->set_next_task(rq, next, SNT_NORMAL); } static inline void @@ -2811,11 +2817,13 @@ static inline void put_prev_set_next_task(struct rq *rq, __put_prev_set_next_dl_server(rq, prev, next); - if (next == prev) + if (next == prev) { + next->sched_class->set_next_task(rq, next, SNT_REPICK); return; + } prev->sched_class->put_prev_task(rq, prev, next); - next->sched_class->set_next_task(rq, next, true); + next->sched_class->set_next_task(rq, next, SNT_PICK); } /* diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index c909ca0d8c87..1e0109ec36b3 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -27,8 +27,11 @@ wakeup_preempt_stop(struct rq *rq, struct task_struct *p, int flags) /* we're never preempted */ } -static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first) +static void set_next_task_stop(struct rq *rq, struct task_struct *stop, enum snt_e type) { + if (type == SNT_REPICK) + return; + stop->se.exec_start = rq_clock_task(rq); } From 819224e506bc7c2d61ec6a58ec6e876b505abce9 Mon Sep 17 00:00:00 2001 From: Kayra Cizmeci Date: Fri, 11 Sep 2026 18:54:48 +0300 Subject: [PATCH 21/64] sched/fair: Remove dead code on enqueue_task_fair() The 'cfs_rq->curr == se' path on enqueue_task_fair() seems to be unachievable. Remove the code that depends on this cfs_rq->curr == se case on enqueue_task_fair() and bool curr too since there are two possibilities and one has been removed, we can proceed directly with the other. Signed-off-by: Kayra Cizmeci Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911155449.1249726-1-kayracizmeci@gmail.com --- kernel/sched/fair.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 36861f435d8a..a03e7f4fc9cd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7988,7 +7988,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) struct sched_entity *se = &p->se; struct cfs_rq *cfs_rq = &rq->cfs; unsigned long weight; - bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -8017,23 +8016,14 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - /* - * XXX comment on the curr thing - */ - curr = (cfs_rq->curr == se); - if (curr) - place_entity(cfs_rq, se, flags); if (se->on_rq && se->sched_delayed) requeue_delayed_entity(cfs_rq, se); weight = enqueue_hierarchy(p, flags); - - if (!curr) { - reweight_eevdf(cfs_rq, se, weight, false); - place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); - __enqueue_entity(cfs_rq, se); - } + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); if (!rq_h_nr_queued && rq->cfs.h_nr_queued) dl_server_start(&rq->fair_server); From fbbc63fed0b09c8c5cf3972db8922ae98406ceec Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 16 Sep 2026 01:31:38 +0900 Subject: [PATCH 22/64] sched/core: Remove redundant core_sched_seq core_sched_seq records whether an rq has consumed the pick made by the last core-wide selection. However, rq->core_pick already carries the same per-rq state. A core-wide selection leaves core_pick populated only for siblings which still need to consume their picks. It is cleared when the current CPU consumes its pick, when a sibling is already running the selected task, or when a pending pick is consumed through the fastpath. The CPU offline path clears it as well. core_task_seq and core_pick_seq serve a separate purpose. core_task_seq changes when the task set changes and for each new core-wide pick, while core_pick_seq records the task sequence on which the selection was made. Their equality therefore establishes that a pending core_pick is still valid. Consequently, once core_pick_seq == core_task_seq, a non-NULL core_pick is sufficient to tell that this rq still has a pick to consume. core_sched_seq duplicates that pending/consumed state. Remove core_sched_seq and use core_pick directly as the pending marker. Testing included an instrumented comparison of the old and new pending predicates under core-scheduling stress. Additional guest tests exercised repeated core-cookie lifecycles, forced idle, and CPU hotplug under load. No predicate divergence or kernel warning was observed. Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260915163138.2973969-1-sh_def@163.com --- kernel/sched/core.c | 10 ++++------ kernel/sched/sched.h | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d368aaf68c8e..ee9b443f760d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6276,10 +6276,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * selection. In this case, do a core-wide selection. */ if (rq->core->core_pick_seq == rq->core->core_task_seq && - rq->core->core_pick_seq != rq->core_sched_seq && rq->core_pick) { - WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); - next = rq->core_pick; rq->dl_server = rq->core_dl_server; rq->core_pick = NULL; @@ -6311,11 +6308,13 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) } /* - * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq + * core->core_task_seq, core->core_pick_seq * * @task_seq guards the task state ({en,de}queues) * @pick_seq is the @task_seq we did a selection on - * @sched_seq is the @pick_seq we scheduled + * + * Once a core-wide selection is committed, a non-NULL core_pick denotes + * a pick which still needs to be consumed on this CPU. * * However, preemptions can cause multiple picks on the same task set. * 'Fix' this by also increasing @task_seq for every pick. @@ -6422,7 +6421,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) rq->core->core_pick_seq = rq->core->core_task_seq; next = rq->core_pick; - rq->core_sched_seq = rq->core->core_pick_seq; /* Something should have been selected for current CPU */ WARN_ON_ONCE(!next); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1b64be5771ca..f7d0b64b9d35 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1371,7 +1371,6 @@ struct rq { struct task_struct *core_pick; struct sched_dl_entity *core_dl_server; unsigned int core_enabled; - unsigned int core_sched_seq; struct rb_root core_tree; /* shared state -- careful with sched_core_cpu_deactivate() */ From abe440b3770fce8eb8ee92a563dfb55a7b9a1c0e Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Thu, 17 Sep 2026 16:39:14 +0100 Subject: [PATCH 23/64] sched/fair: Drop idle recency from slow-path CPU selection The slow-path CPU picker favours the most recently idle CPU as a proxy for cache warmth. A more recent idle stamp may make ongoing entry more likely. If entry cannot be aborted, that CPU must finish entry and then exit, potentially paying more than an already-resident CPU with the same advertised exit latency. The advertised worst-case latency covers both cases. idle_stamp does not timestamp the current CPUIdle entry, so this is only a heuristic. A recent scheduler-idle transition may also mark a short gap in recurring task activity, making the CPU likely to be busy again soon. Drop the timestamp tie-break and retain the first candidate unless a lower advertised exit latency is found. Signed-off-by: Christian Loehle Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260917153915.1563875-2-christian.loehle@arm.com --- kernel/sched/fair.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a03e7f4fc9cd..c4d4b55a09b1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8444,7 +8444,6 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct * { unsigned long load, min_load = ULONG_MAX; unsigned int min_exit_latency = UINT_MAX; - u64 latest_idle_timestamp = 0; int least_loaded_cpu = this_cpu; int shallowest_idle_cpu = -1; int i; @@ -8466,22 +8465,10 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct * if (available_idle_cpu(i)) { struct cpuidle_state *idle = idle_get_state(rq); if (idle && idle->exit_latency < min_exit_latency) { - /* - * We give priority to a CPU whose idle state - * has the smallest exit latency irrespective - * of any idle timestamp. - */ min_exit_latency = idle->exit_latency; - latest_idle_timestamp = rq->idle_stamp; shallowest_idle_cpu = i; } else if ((!idle || idle->exit_latency == min_exit_latency) && - rq->idle_stamp > latest_idle_timestamp) { - /* - * If equal or no active idle state, then - * the most recently idled CPU might have - * a warmer cache. - */ - latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu == -1) { shallowest_idle_cpu = i; } } else if (shallowest_idle_cpu == -1) { From c9ce69fc43bd07dbed9d23a504e56f84604ecf42 Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Thu, 17 Sep 2026 16:39:15 +0100 Subject: [PATCH 24/64] sched/fair: Randomize equally shallow slow-path candidates Picking the first eligible idle CPU leaves a scan-order bias. Concurrent slow-path selectors can choose the same CPU before either task is enqueued. Use reservoir sampling for equal exit latencies, resetting the candidate count when a shallower candidate appears. Use the per-CPU scheduler PRNG and reciprocal_scale() to avoid variable division or a second scan. Use a u64 latency key with U64_MAX for unpublished states. Published states take precedence; when none are found, sample among the idle CPUs without a published state. Signed-off-by: Christian Loehle Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260917153915.1563875-3-christian.loehle@arm.com --- kernel/sched/fair.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c4d4b55a09b1..2f5c7063b032 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -8443,7 +8444,8 @@ static int sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { unsigned long load, min_load = ULONG_MAX; - unsigned int min_exit_latency = UINT_MAX; + u64 min_exit_latency = U64_MAX; + unsigned int nr_candidates = 0; int least_loaded_cpu = this_cpu; int shallowest_idle_cpu = -1; int i; @@ -8464,12 +8466,16 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct * if (available_idle_cpu(i)) { struct cpuidle_state *idle = idle_get_state(rq); - if (idle && idle->exit_latency < min_exit_latency) { - min_exit_latency = idle->exit_latency; - shallowest_idle_cpu = i; - } else if ((!idle || idle->exit_latency == min_exit_latency) && - shallowest_idle_cpu == -1) { + u64 exit_latency = idle ? idle->exit_latency : U64_MAX; + + if (shallowest_idle_cpu == -1 || exit_latency < min_exit_latency) { + min_exit_latency = exit_latency; shallowest_idle_cpu = i; + nr_candidates = 1; + } else if (exit_latency == min_exit_latency) { + nr_candidates++; + if (!reciprocal_scale(sched_rng(), nr_candidates)) + shallowest_idle_cpu = i; } } else if (shallowest_idle_cpu == -1) { load = cpu_load(cpu_rq(i)); From aae2a33ea66299f39252bc1891bb4f05f631255d Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 21 Sep 2026 17:22:31 +0200 Subject: [PATCH 25/64] sched/eevdf: Ensure that vprot will never go above a min slice ineligible_vruntime() can be after the duration of a min slice but we want to make sure that current will not run more than a min slice. Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260921152238.3804392-2-vincent.guittot@linaro.org --- kernel/sched/fair.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2f5c7063b032..0969be26c4d6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1128,10 +1128,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity /* If there are shorter slices than se's one */ if (slice != se->slice) { + vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); if (sched_feat(PREEMPT_SHORT)) vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); - else - vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); } se->vprot = vprot; From 4bf32ec3327d2d2286e96508cd128ce1592f43ce Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 21 Sep 2026 17:22:32 +0200 Subject: [PATCH 26/64] sched/eevdf: Align update_protect_slice to set_protect_slice Apply the same logic as for set_protect_slice when updating it. Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260921152238.3804392-3-vincent.guittot@linaro.org --- kernel/sched/fair.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0969be26c4d6..82155d91de06 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -821,12 +821,6 @@ static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) if (curr && !curr->on_rq) curr = NULL; - /* - * This is called from set_next_task_fair(.first=true) / - * set_protect_slice() so curr had better be set and on_rq. - */ - WARN_ON_ONCE(!curr); - if (weight) { s64 runtime = cfs_rq->sum_w_vruntime; @@ -1138,10 +1132,19 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 slice = cfs_rq_min_slice(cfs_rq); u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); + u64 slice = normalized_sysctl_sched_base_slice; + u64 vprot; - se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); + if (sched_feat(RUN_TO_PARITY)) + slice = cfs_rq_min_slice(cfs_rq); + + vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); + + if (sched_feat(PREEMPT_SHORT) && slice != se->slice) + vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); + + se->vprot = vprot; } static inline bool protect_slice(struct sched_entity *se) From d2e0100827578641df2f85546bdb57bb1d8ff55b Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 21 Sep 2026 17:22:33 +0200 Subject: [PATCH 27/64] sched/eevdf: Handle more short slice waking cases Handle better the cases with several short slice tasks waking up simultaneously. Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260921152238.3804392-4-vincent.guittot@linaro.org --- kernel/sched/fair.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 82155d91de06..03206e15e6fe 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9836,8 +9836,14 @@ static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) { - if (cfs_rq->next && cfs_rq->next->slice < pse->slice) - return false; + if (cfs_rq->next) { + if (cfs_rq->next->slice < pse->slice) + return false; + + if (cfs_rq->next->slice == pse->slice && + entity_before(cfs_rq->next, pse)) + return false; + } set_next_buddy(cfs_rq, pse); return true; From 627ea30aca3b309af40cdd07a35eb1027ba4a0bc Mon Sep 17 00:00:00 2001 From: "Shubhang Kaushik (Ampere)" Date: Tue, 22 Sep 2026 14:57:05 -0700 Subject: [PATCH 28/64] sched/wait: Clarify WF_SYNC wakeup semantics The synchronous waitqueue wakeup comments currently state that a synchronous wakee will not be migrated to another CPU. This is not guaranteed by the scheduler wakeup path. WF_SYNC is an advisory hint that the caller expects the waker to schedule away soon. Scheduler classes may use it for placement or preemption, but callers must not rely on it to prevent migration, preserve CPU locality, or make the wakee run next. Keep this contract next to the flag definition, remove the stale waitqueue wording, and make the locked helper refer to the unlocked variant. Signed-off-by: Shubhang Kaushik (Ampere) Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260922-sched-wf-sync-doc-v3-1-23ebe9e27bef@gentwo.org --- kernel/sched/sched.h | 9 +++++++-- kernel/sched/wait.c | 22 +++++----------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f7d0b64b9d35..b98084e1f5b0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2535,8 +2535,13 @@ static inline int task_on_rq_migrating(struct task_struct *p) #define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ #define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ #define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ - -#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ +/* + * Hint that the caller expects the waker to sleep soon. + * Scheduler classes may use it for placement or preemption. + * Callers must not rely on it to prevent migration, + * preserve CPU locality or make the wakee run next. + */ +#define WF_SYNC 0x10 #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ #define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ #define WF_RQ_SELECTED 0x80 /* ->select_task_rq() was called */ diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index d033f600f48c..477e4bf9c01e 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -174,15 +174,11 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key); * @mode: which threads * @key: opaque value to be passed to wakeup targets * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. + * Passes WF_SYNC to waitqueue wake functions. The default wake function + * forwards it to the scheduler; see WF_SYNC for the hint's semantics. * - * On UP it can prevent extra preemption. - * - * If this function wakes up a task, it executes a full memory barrier before - * accessing the task state. + * If this function wakes up a task, it executes a full memory barrier + * before accessing the task state. */ void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) @@ -200,15 +196,7 @@ EXPORT_SYMBOL_GPL(__wake_up_sync_key); * @mode: which threads * @key: opaque value to be passed to wakeup targets * - * The sync wakeup differs in that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - * - * If this function wakes up a task, it executes a full memory barrier before - * accessing the task state. + * Same as __wake_up_sync_key(), but called with @wq_head->lock held. */ void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) From 313b652837d0541684680a7d07a95c5560441d4f Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:40 +0200 Subject: [PATCH 29/64] sched/core: Drop mutex locks before proxy rescheduling find_proxy_task() can call proxy_resched_idle() from return expressions inside mutex wait_lock and blocked task lock guard scopes. Return expressions are evaluated before scoped guards are released, so proxy_resched_idle() invokes scheduling-class callbacks while both raw spinlocks are held. Route these cases through a common label outside the guard scopes. The common label is required because a direct return would evaluate proxy_resched_idle() before guard cleanup and run the class callbacks under the proxy-exec locks again. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Link: https://patch.msgid.link/20260922165445.943315-2-arighi@nvidia.com --- kernel/sched/core.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ee9b443f760d..f34e73cde87d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6980,7 +6980,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) { /* XXX Don't handle blocked owners/delayed dequeue yet */ if (curr_in_chain) - return proxy_resched_idle(rq); + goto resched_idle; __clear_task_blocked_on(p, NULL); goto deactivate; } @@ -6992,7 +6992,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * and leave that CPU to sort things out. */ if (curr_in_chain) - return proxy_resched_idle(rq); + goto resched_idle; goto migrate_task; } @@ -7005,7 +7005,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * case we should end up back in find_proxy_task(), this time * hopefully with all relevant tasks already enqueued. */ - return proxy_resched_idle(rq); + goto resched_idle; } /* @@ -7042,7 +7042,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * So schedule rq->idle so that ttwu_runnable() can get the rq * lock and mark owner as running. */ - return proxy_resched_idle(rq); + goto resched_idle; } /* * OK, now we're absolutely sure @owner is on this @@ -7062,6 +7062,8 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) } return owner; +resched_idle: + return proxy_resched_idle(rq); deactivate: proxy_deactivate(rq, p); return NULL; From 8f8c0417e97382c3473ef56b0cd472badb40e4a0 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:41 +0200 Subject: [PATCH 30/64] sched/core: Dequeue waking proxy donors before reset proxy_needs_return() resets an active donor while holding blocked_lock. proxy_reset_donor() invokes scheduling-class callbacks, adding an unnecessary raw-spinlock nesting. For an EXT donor, resetting rq->donor first calls put_prev_task_scx() while SCX_TASK_QUEUED and is_blocked are still set. That path reenqueues the retained donor through scx_do_enqueue_task(), only for the following block_task() to dequeue it again. Split block_task() so the waking donor is first dequeued from its scheduling class while it is still rq->donor. This lets dequeue_task_scx() end the donor's running session and clear SCX_TASK_QUEUED before proxy_reset_donor() invokes put_prev_task_scx(). Keep the generic on_rq state set until donor references are replaced, then complete the runqueue removal. This follows the normal sleep ordering, avoids transiently re-enqueuing the waking donor, and moves the scheduling-class callbacks outside blocked_lock. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260922165445.943315-3-arighi@nvidia.com --- kernel/sched/core.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f34e73cde87d..07cc17141afe 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2252,7 +2252,8 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) dequeue_task(rq, p, flags); } -static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state) +static bool dequeue_block_task(struct rq *rq, struct task_struct *p, + unsigned long task_state) { int flags = DEQUEUE_NOCLOCK; @@ -2273,9 +2274,15 @@ static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_ * * Where __schedule() and ttwu() have matching control dependencies. * - * After this, schedule() must not care about p->state any more. + * Once the caller invokes __block_task(), schedule() must not care about + * p->state any more. */ - if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) + return dequeue_task(rq, p, DEQUEUE_SLEEP | flags); +} + +static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state) +{ + if (dequeue_block_task(rq, p, task_state)) __block_task(rq, p); } @@ -3781,6 +3788,8 @@ static inline void proxy_reset_donor(struct rq *rq) */ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) { + bool dequeued; + /* * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu. * @@ -3803,12 +3812,23 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) /* If already current, don't need to return migrate */ if (task_current(rq, p)) return false; - - /* If we're return migrating the rq->donor, switch it out for idle */ - if (task_current_donor(rq, p)) - proxy_reset_donor(rq); } - block_task(rq, p, TASK_WAKING); + + dequeued = dequeue_block_task(rq, p, TASK_WAKING); + + /* + * Dequeue @p from its scheduling class before resetting rq->donor. + * In particular, sched_ext needs to end the donor's running session + * and clear SCX_TASK_QUEUED before put_prev_task_scx() is called by + * proxy_reset_donor(); otherwise it would reenqueue the blocked donor. + * + * Keep on_rq set until all donor references have been replaced. + */ + if (task_current_donor(rq, p)) + proxy_reset_donor(rq); + + if (dequeued) + __block_task(rq, p); return true; } #else /* !CONFIG_SCHED_PROXY_EXEC */ From a49653d0abebee32bf3ecf650fae9f9dd8298135 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:42 +0200 Subject: [PATCH 31/64] sched/core: Mark wakeups completed through ttwu_runnable() ttwu_runnable() completes a wakeup for a task which remains queued without calling activate_task() or its scheduling-class enqueue callback. Scheduling classes currently cannot distinguish this path from wakeups which performed a full activation. Add WF_TTWU_RQ to identify wakeups completed through ttwu_runnable(). A later sched_ext change uses it to reconsider retained proxy donors which wake without another ops.enqueue() call. No functional change. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260922165445.943315-4-arighi@nvidia.com --- kernel/sched/core.c | 2 +- kernel/sched/sched.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 07cc17141afe..6b42f916a36e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3919,7 +3919,7 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) * When on_rq && !on_cpu the task is preempted, see if * it should preempt the task that is current now. */ - wakeup_preempt(rq, p, wake_flags); + wakeup_preempt(rq, p, wake_flags | WF_TTWU_RQ); } ttwu_do_wakeup(p); return 1; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b98084e1f5b0..a0f391fc4294 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2545,6 +2545,7 @@ static inline int task_on_rq_migrating(struct task_struct *p) #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ #define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ #define WF_RQ_SELECTED 0x80 /* ->select_task_rq() was called */ +#define WF_TTWU_RQ 0x100 /* Wakeup completed through ttwu_runnable() */ static_assert(WF_EXEC == SD_BALANCE_EXEC); static_assert(WF_FORK == SD_BALANCE_FORK); From 57c75e3ae38c40acb5882a57c36f2d5234e066d0 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:43 +0200 Subject: [PATCH 32/64] sched: Add helper to block retained proxy donors Scheduler ownership changes may need to turn a mutex-blocked task retained on the runqueue for proxy execution back into a normally blocked task. Add sched_proxy_block_task() to perform that transition while holding p->pi_lock and the task's runqueue lock. Reset an active donor to the physical current task before blocking it. If DELAY_DEQUEUE keeps a FAIR donor queued, the immediately following sched_change_begin() completes the dequeue through switching_from_fair() before recording its queued state. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260922165445.943315-5-arighi@nvidia.com --- kernel/sched/core.c | 28 ++++++++++++++++++++++++++++ kernel/sched/sched.h | 6 ++++++ 2 files changed, 34 insertions(+) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6b42f916a36e..35374e0240c3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6834,6 +6834,34 @@ static void proxy_deactivate(struct rq *rq, struct task_struct *donor) block_task(rq, donor, state); } +/* + * Remove a retained proxy donor before changing its scheduler ownership. + * The caller holds p->pi_lock, so p cannot wake and migrate if block_task() + * drops it from the runqueue. If DELAY_DEQUEUE keeps a blocked fair task + * queued, switching_from_fair() completes the dequeue in the immediately + * following sched_change_begin(). + */ +void sched_proxy_block_task(struct rq *rq, struct task_struct *p) +{ + unsigned long state = READ_ONCE(p->__state); + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(rq); + + if (!p->is_blocked || !task_on_rq_queued(p)) + return; + if (WARN_ON_ONCE(state == TASK_RUNNING)) + return; + + if (task_current_donor(rq, p)) + proxy_reset_donor(rq); + + if (!p->se.sched_delayed) + block_task(rq, p, state); + + WARN_ON_ONCE(task_on_rq_queued(p) && !p->se.sched_delayed); +} + static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf) __releases(__rq_lockp(rq)) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a0f391fc4294..d41d23a5632b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2516,6 +2516,12 @@ static inline bool task_is_blocked(struct task_struct *p) return !!p->blocked_on; } +#ifdef CONFIG_SCHED_PROXY_EXEC +void sched_proxy_block_task(struct rq *rq, struct task_struct *p); +#else +static inline void sched_proxy_block_task(struct rq *rq, struct task_struct *p) {} +#endif + static inline int task_on_cpu(struct rq *rq, struct task_struct *p) { return p->on_cpu; From be100c77178ef2299a080eb698720832a870248b Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:44 +0200 Subject: [PATCH 33/64] sched: Add sched_ext hooks for proxy execution Proxy execution splits the scheduling context (the donor) from the execution context (the lock owner). sched_ext needs to observe that split at three points in __schedule(): - whether a blocked EXT task can be retained on the runqueue as a donor, - when a donor's scheduling context starts driving a lock owner, - after proxy resolution, when deferred reenqueue work that was blocked by an active proxy relationship can be retried. Introduce scx_allow_proxy_exec(), scx_proxy_donor_start() and scx_proxy_reenqueue_retry(), and add their call sites in __schedule(). The implementations are empty here and are filled in by the sched_ext changes that follow, so that all the sched core changes needed by proxy execution stay together in the preparatory patches. SCHED_PROXY_EXEC still depends on !SCHED_CLASS_EXT, so the new hooks are inert: they are compiled out with CONFIG_SCHED_CLASS_EXT=n and unreachable otherwise. This is a preparatory change to support proxy execution with sched_ext. No functional change. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260922165445.943315-6-arighi@nvidia.com --- kernel/sched/core.c | 12 +++++++----- kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/ext.h | 7 +++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 35374e0240c3..acc89310d488 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7243,13 +7243,12 @@ static void __sched notrace __schedule(int sched_mode) } } else if (!preempt && prev_state) { /* - * We pass task_is_blocked() as the should_block arg - * in order to keep mutex-blocked tasks on the runqueue - * for slection with proxy-exec (without proxy-exec - * task_is_blocked() will always be false). + * Keep mutex-blocked tasks on the runqueue for proxy execution + * only when their scheduling class allows it. Without proxy + * execution, task_is_blocked() always returns false. */ try_to_block_task(rq, prev, &prev_state, - !task_is_blocked(prev)); + !task_is_blocked(prev) || !scx_allow_proxy_exec(prev)); switch_count = &prev->nvcsw; } @@ -7270,6 +7269,7 @@ static void __sched notrace __schedule(int sched_mode) } if (next == rq->idle) { zap_balance_callbacks(rq); + scx_proxy_reenqueue_retry(rq, next); goto keep_resched; } } @@ -7290,6 +7290,8 @@ static void __sched notrace __schedule(int sched_mode) donor->sched_class->put_prev_task(rq, donor, donor); donor->sched_class->set_next_task(rq, donor, SNT_PICK); } + scx_proxy_donor_start(rq); + scx_proxy_reenqueue_retry(rq, next); } else { rq_set_donor(rq, next); } diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 53275bc7c029..3f7915646779 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,6 +24,11 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); +bool scx_allow_proxy_exec(const struct task_struct *p) +{ + return true; +} + /* * NOTE: sched_ext is in the process of growing multiple scheduler support and * scx_root usage is in a transitional state. Naked dereferences are safe if the @@ -1079,6 +1084,10 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) +{ +} + void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq) { @@ -3065,6 +3074,10 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t } } +void scx_proxy_donor_start(struct rq *rq) +{ +} + static enum scx_cpu_preempt_reason preempt_reason_from_class(const struct sched_class *class) { diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 0b7fc46aee08..3cfbfeb1bf9d 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -20,6 +20,9 @@ void scx_rq_deactivate(struct rq *rq); int scx_check_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); +bool scx_allow_proxy_exec(const struct task_struct *p); +void scx_proxy_donor_start(struct rq *rq); +void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next); void init_sched_ext_class(void); static inline u32 scx_cpuperf_target(s32 cpu) @@ -54,6 +57,10 @@ static inline void scx_rq_deactivate(struct rq *rq) {} static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } +static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; } +static inline void scx_proxy_donor_start(struct rq *rq) {} +static inline void scx_proxy_reenqueue_retry(struct rq *rq, + struct task_struct *next) {} static inline void init_sched_ext_class(void) {} #endif /* CONFIG_SCHED_CLASS_EXT */ From a8d0854a76a876ea474e17322acb680716a09da2 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:16 +0530 Subject: [PATCH 34/64] sched/cputime: Add kcpustat_field_total helper Provide a new helper function which sums up a given type of cpustat over a specified cpumask. This allows the caller's code to be simpler and avoids duplication. For example, subsequent patch in the steal governor use this exact same pattern when calculating steal time. Number of cpus can be derived from cpumask_weight() where necessary. Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yury Norov Reviewed-by: Mete Durlu Acked-by: Frederic Weisbecker Link: https://patch.msgid.link/20260928053728.797539-2-sshegde@linux.ibm.com --- arch/s390/kernel/hiperdispatch.c | 10 +++------- fs/proc/uptime.c | 6 +----- include/linux/kernel_stat.h | 11 +++++++++++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 217206522266..c21496f0a141 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -207,16 +207,12 @@ static unsigned long hd_calculate_steal_percentage(void) { unsigned long time_delta, steal_delta, steal, percentage; static ktime_t prev; - int cpus, cpu; + int cpus; ktime_t now; - cpus = 0; - steal = 0; percentage = 0; - for_each_cpu(cpu, &hd_vmvl_cpumask) { - steal += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - cpus++; - } + steal = kcpustat_field_total(CPUTIME_STEAL, &hd_vmvl_cpumask); + cpus = cpumask_weight(&hd_vmvl_cpumask); /* * If there is no vertical medium and low CPUs steal time * is 0 as vertical high CPUs shouldn't experience steal time. diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c index 433aa947cd57..53143c66cbe1 100644 --- a/fs/proc/uptime.c +++ b/fs/proc/uptime.c @@ -15,12 +15,8 @@ static int uptime_proc_show(struct seq_file *m, void *v) struct timespec64 idle; u64 idle_nsec; u32 rem; - int i; - - idle_nsec = 0; - for_each_possible_cpu(i) - idle_nsec += kcpustat_field(CPUTIME_IDLE, i); + idle_nsec = kcpustat_field_total(CPUTIME_IDLE, cpu_possible_mask); ktime_get_boottime_ts64(&uptime); timens_add_boottime(&uptime); diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 9ca6c2259dfe..c1e85550bf12 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -196,6 +196,17 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) } #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ +static inline u64 kcpustat_field_total(enum cpu_usage_stat usage, const struct cpumask *cpus) +{ + u64 total = 0; + int cpu; + + for_each_cpu(cpu, cpus) + total += kcpustat_field(usage, cpu); + + return total; +} + extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); extern void account_system_time(struct task_struct *, int, u64); From cfb463b7172dca2f29b2221b71b54890f268e3fb Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:17 +0530 Subject: [PATCH 35/64] cpumask: Introduce cpumask_intersects_and Introduce bitmap_intersects_and() to determine whether the intersection of three bitmaps is non-empty. Unlike cpumask_first_and_and(), this returns immediately when an intersecting word is found and does not calculate the first matching bit. Add cpumask_intersects_and() as the corresponding cpumask wrapper. A subsequent patch uses the helper to determine whether a task has a CPU that is present in its affinity mask, the preferred CPU mask, and task possible CPU mask. Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Reviwed-by: Yury Norov Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yury Norov Link: https://patch.msgid.link/20260928053728.797539-3-sshegde@linux.ibm.com --- include/linux/bitmap.h | 14 ++++++++++++++ include/linux/cpumask.h | 18 ++++++++++++++++++ lib/bitmap.c | 17 +++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 7df1573a409c..adafbcf2016b 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -52,6 +52,7 @@ struct device; * bitmap_complement(dst, src, nbits) *dst = ~(*src) * bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal? * bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap? + * bitmap_intersects_and(src1, src2, src3, nbits) Do *src1, *src2 and *src3 overlap? * bitmap_subset(src1, src2, nbits) Is *src1 a subset of *src2? * bitmap_empty(src, nbits) Are all bits zero in *src? * bitmap_full(src, nbits) Are all bits set in *src? @@ -181,6 +182,9 @@ void __bitmap_replace(unsigned long *dst, const unsigned long *mask, unsigned int nbits); bool __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +bool __bitmap_intersects_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, unsigned int nbits); bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); @@ -445,6 +449,16 @@ bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, uns return __bitmap_intersects(src1, src2, nbits); } +static __always_inline +bool bitmap_intersects_and(const unsigned long *src1, const unsigned long *src2, + const unsigned long *src3, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return ((*src1 & *src2 & *src3) & BITMAP_LAST_WORD_MASK(nbits)) != 0; + else + return __bitmap_intersects_and(src1, src2, src3, nbits); +} + static __always_inline bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 4c8bb6953107..7c8f16797f94 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -824,6 +824,24 @@ bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p small_cpumask_bits); } +/** + * cpumask_intersects_and - (*src1p & *src2p & *src3p) != 0 + * @src1p: the first input + * @src2p: the second input + * @src3p: the third input + * + * Return: true if AND of the three cpumasks is non-empty, + * otherwise false + */ +static __always_inline +bool cpumask_intersects_and(const struct cpumask *src1p, + const struct cpumask *src2p, + const struct cpumask *src3p) +{ + return bitmap_intersects_and(cpumask_bits(src1p), cpumask_bits(src2p), + cpumask_bits(src3p), small_cpumask_bits); +} + /** * cpumask_subset - (*src1p & ~*src2p) == 0 * @src1p: the first input diff --git a/lib/bitmap.c b/lib/bitmap.c index ed685127a107..d1cb8a507c60 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -308,6 +308,23 @@ bool __bitmap_intersects(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_intersects); +bool __bitmap_intersects_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, unsigned int bits) +{ + unsigned int k, lim = bits / BITS_PER_LONG; + + for (k = 0; k < lim; ++k) + if (bitmap1[k] & bitmap2[k] & bitmap3[k]) + return true; + + if (bits % BITS_PER_LONG) + if ((bitmap1[k] & bitmap2[k] & bitmap3[k]) & BITMAP_LAST_WORD_MASK(bits)) + return true; + return false; +} +EXPORT_SYMBOL(__bitmap_intersects_and); + bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { From 06a49ef784acf91cee78d50c5307d400f86ccafb Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:18 +0530 Subject: [PATCH 36/64] sched/docs: Document cpu_preferred_mask and Preferred CPU concept Add documentation for new CPU state called preferred CPU state and corresponding cpumask called cpu_preferred_mask. This could help users in understanding what it is and how to use it. Document the role of scheduler and driver for this feature to work. Details regarding the driver documentation will be added in later patches under Documentation/driver-api/steal-governor.rst. Newly added file could be used for other paravirt usecase documentation. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-4-sshegde@linux.ibm.com --- Documentation/scheduler/index.rst | 1 + Documentation/scheduler/sched-paravirt.rst | 67 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 Documentation/scheduler/sched-paravirt.rst diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst index 17ce8d76befc..a43647b9706d 100644 --- a/Documentation/scheduler/index.rst +++ b/Documentation/scheduler/index.rst @@ -23,5 +23,6 @@ Scheduler sched-stats sched-ext sched-debug + sched-paravirt text_files diff --git a/Documentation/scheduler/sched-paravirt.rst b/Documentation/scheduler/sched-paravirt.rst new file mode 100644 index 000000000000..311cdbf2722b --- /dev/null +++ b/Documentation/scheduler/sched-paravirt.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. _sched-paravirt: + +Preferred CPUs +============== + +In paravirtualized environments CPU overcommit is a common scenario. +i.e. the sum of virtual CPUs (vCPUs) of all VMs is greater than number of +physical CPUs (pCPUs). Under such conditions when all or many VMs have +high utilization, hypervisor won't be able to satisfy the CPU requirement +and has to context switch within or across VMs. The hypervisor needs to +preempt one vCPU to run another. This is called vCPU preemption. +This is more expensive compared to task context switch within a vCPU, since +hypervisor lacks vCPU context and could preempt a critical section which +slows forward progress. + +In such cases it is better that combined vCPU demand from all VMs is reduced +by not using some of the vCPUs in each VM. vCPUs where workload can be safely +scheduled which won't increase any contention for pCPU are called +"Preferred CPUs". + +One of the main design constructs is that preferred CPUs are always +a subset of active CPUs. In most cases preferred CPUs will be same as +active CPUs. When there is pCPU contention, Preferred CPUs will reduce +based on the steal time. When the pCPU contention goes away as indicated +by steal time, Preferred CPUs could become same as active CPUs again. +The policy decisions are to be taken by driver. +For example, steal_governor. Look at its documentation for more +details. (``drivers/virt/steal_governor.c``) + +Scheduling decisions such as wakeup, pushing the task etc, need this +CPU state info. This is maintained in ``cpu_preferred_mask``. +vCPUs which are not in ``cpu_preferred_mask`` should be treated as vCPUs which +should not be used at this moment provided it doesn't break user affinity. + +This is achieved by: + +1. Selecting a preferred CPU at wakeup using fallback mechanism. +2. Pushing the task away from non-preferred CPU at tick. +3. Selecting only preferred CPUs for load balance. + +``/sys/devices/system/cpu/preferred`` prints the current ``cpu_preferred_mask`` +in cpulist format. + +Notes: + +1. This feature is available under ``CONFIG_PREFERRED_CPU``. Driver which + makes decisions should enable it. For example, steal_governor driver + (``CONFIG_STEAL_GOVERNOR``). On enabling the driver, CPU preferred state + can change based on steal time. Without the driver, preferred CPUs is + same as active CPUs. + +2. This feature works for the FAIR class only. + +3. A pinned task, which can't be moved to preferred CPUs will continue + to run based on its affinity. But no load balancing happens if it is affined + only on non-preferred CPUs. + +4. Decision to change the preferred CPU state is driven by the kernel. + Hence it shouldn't break user affinities. One of the main reasons why + CPU hotplug or Isolated cpuset partitions was not a solution. + +5. This feature works best only when all the Guest VMs enable the feature as + it is a co-operative scheme. If a specific VM doesn't enable this feature + it may end up with more CPUs than others, still should lead to better + performance when seen from system view. + Users who enable this driver must ensure it is enabled in all Guest VMs. From 518b32bd5bb373aa334e4b73d359689071d50b5c Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:19 +0530 Subject: [PATCH 37/64] cpumask: Introduce cpu_preferred_mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide the preferred CPU infrastructure. Define get/set macros which could be used to get/set CPU state as preferred. CONFIG_PREFERRED_CPU will be selected by the driver which handles steal time values. It is going to set/clear preferred CPU state. This driver will be called steal_governor and it is introduced in subsequent patches. It periodically computes the steal ratio and decides on preferred CPU state. A CPU is set to preferred when it becomes active. Later it may be marked as non-preferred depending on steal ratio by the steal_governor. Always maintain design construct of preferred is subset of active. i.e. preferred ⊆ active ⊆ online ⊆ present ⊆ possible With CONFIG_PREFERRED_CPU=n, ensure set_cpu_preferred is a nop and get method returns the active state in that case. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-5-sshegde@linux.ibm.com --- include/linux/cpumask.h | 24 ++++++++++++++++++++++++ kernel/Kconfig.preempt | 4 ++++ kernel/cpu.c | 6 ++++++ kernel/sched/core.c | 5 +++++ 4 files changed, 39 insertions(+) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 7c8f16797f94..bf89bb3f30f6 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -121,12 +121,20 @@ extern struct cpumask __cpu_enabled_mask; extern struct cpumask __cpu_present_mask; extern struct cpumask __cpu_active_mask; extern struct cpumask __cpu_dying_mask; + +#ifdef CONFIG_PREFERRED_CPU +extern struct cpumask __cpu_preferred_mask; +#else +#define __cpu_preferred_mask __cpu_active_mask +#endif + #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) #define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask) #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) +#define cpu_preferred_mask ((const struct cpumask *)&__cpu_preferred_mask) extern atomic_t __num_online_cpus; extern unsigned int __num_possible_cpus; @@ -1181,6 +1189,12 @@ void init_cpu_possible(const struct cpumask *src); #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) +#ifdef CONFIG_PREFERRED_CPU +#define set_cpu_preferred(cpu, preferred) assign_cpu((cpu), &__cpu_preferred_mask, (preferred)) +#else +#define set_cpu_preferred(cpu, preferred) do { } while (0) +#endif + void set_cpu_online(unsigned int cpu, bool online); void set_cpu_possible(unsigned int cpu, bool possible); @@ -1275,6 +1289,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return cpumask_test_cpu(cpu, cpu_dying_mask); } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpumask_test_cpu(cpu, cpu_preferred_mask); +} + #else #define num_online_cpus() 1U @@ -1313,6 +1332,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return false; } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpu == 0; +} + #endif /* NR_CPUS > 1 */ #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 985aea617cfe..edc067a0c422 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -194,3 +194,7 @@ config SCHED_CLASS_EXT For more information: Documentation/scheduler/sched-ext.rst https://github.com/sched-ext/scx + +config PREFERRED_CPU + bool + depends on SMP && PARAVIRT diff --git a/kernel/cpu.c b/kernel/cpu.c index b3c8553d7bd6..376d297a6292 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -3103,6 +3103,11 @@ EXPORT_SYMBOL(__cpu_dying_mask); atomic_t __num_online_cpus __read_mostly; EXPORT_SYMBOL(__num_online_cpus); +#ifdef CONFIG_PREFERRED_CPU +struct cpumask __cpu_preferred_mask __read_mostly; +EXPORT_SYMBOL_GPL(__cpu_preferred_mask); +#endif + void init_cpu_present(const struct cpumask *src) { cpumask_copy(&__cpu_present_mask, src); @@ -3160,6 +3165,7 @@ void __init boot_cpu_init(void) /* Mark the boot cpu "present", "online" etc for SMP and UP case */ set_cpu_online(cpu, true); set_cpu_active(cpu, true); + set_cpu_preferred(cpu, true); set_cpu_present(cpu, true); set_cpu_possible(cpu, true); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index acc89310d488..d4edee72d793 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8632,6 +8632,9 @@ int sched_cpu_activate(unsigned int cpu) */ sched_set_rq_online(rq, cpu); + /* preferred is subset of active and follows its state */ + set_cpu_preferred(cpu, true); + return 0; } @@ -8645,6 +8648,8 @@ int sched_cpu_deactivate(unsigned int cpu) if (ret) return ret; + set_cpu_preferred(cpu, false); + /* * Remove CPU from nohz.idle_cpus_mask to prevent participating in * load balancing when not active From 62082451655747dae25496d126e58f985df29f50 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:20 +0530 Subject: [PATCH 38/64] sysfs: Add preferred CPU file Add a "preferred" file in /sys/devices/system/cpu/ when kernel is built with CONFIG_PREFERRED_CPU=y. This would help - Users to quickly check which CPUs are marked as preferred. - Userspace daemons such as irqbalance to use this mask to route irqs into preferred CPUs. For example: cat /sys/devices/system/cpu/online 0-719 cat /sys/devices/system/cpu/preferred 0-599 <<< Implies 0-599 are preferred for workloads and 600-719 should be avoided at this moment. cat /sys/devices/system/cpu/preferred 0-719 <<< All CPUs are usable. There is no preference. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-6-sshegde@linux.ibm.com --- Documentation/ABI/testing/sysfs-devices-system-cpu | 14 ++++++++++++++ drivers/base/cpu.c | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 82d10d556cc8..8f250b8693b8 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -806,3 +806,17 @@ Date: Nov 2022 Contact: Linux kernel mailing list Description: (RO) the list of CPUs that can be brought online. + +What: /sys/devices/system/cpu/preferred +Date: Sep 2026 +Contact: Linux kernel mailing list +Description: + (RO) the list of preferred CPUs applicable in + paravirtualized environments. + + The steal governor driver dynamically adjusts this mask + based on observed steal time. Scheduling tasks on + CPUs outside of this list may lead to performance + degradations due to underlying physical CPU contention. + + See Documentation/scheduler/sched-paravirt.rst for more details. diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 69e52fed4241..747915ff974f 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -391,6 +391,15 @@ static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env) } #endif +#ifdef CONFIG_PREFERRED_CPU +static ssize_t preferred_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_preferred_mask)); +} +static DEVICE_ATTR_RO(preferred); +#endif + const struct bus_type cpu_subsys = { .name = "cpu", .dev_name = "cpu", @@ -531,6 +540,9 @@ static struct attribute *cpu_root_attrs[] = { #endif #ifdef CONFIG_GENERIC_CPU_AUTOPROBE &dev_attr_modalias.attr, +#endif +#ifdef CONFIG_PREFERRED_CPU + &dev_attr_preferred.attr, #endif NULL }; From d8a3da0de8433d494db1ff5ec541559c764da4ab Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:21 +0530 Subject: [PATCH 39/64] sched/core: Try to use a preferred CPU in is_cpu_allowed When possible, try to choose a preferred CPU. This is essential to maintain user affinities when preferred CPUs change. A task pinned on a non-preferred CPU should continue to run there, since this is a non-user triggered event. If a CPU is non-preferred and the task can run on other CPUs which are currently preferred, then choose a preferred CPU instead. This is decided by checking if cpus_ptr, cpu_preferred_mask and task possible CPUs intersect or not. If yes, then the task has other preferred CPUs. This takes care of tasks with architecture-specific CPU masks (e.g., 32-bit tasks on arm64). The push task mechanism uses a stopper thread which calls select_fallback_rq() and uses this mechanism to pick a preferred CPU. This takes care of the wakeup path for FAIR tasks too. is_cpu_allowed() is called to ensure wakeups happen on preferred CPUs. With that, additional checks in available_idle_cpu() are not necessary. Ignore the preferred CPU state if a task's affinity is changing and its new mask no longer includes the CPU it is currently running on. This ensures migration_cpu_stop() does not abort, preventing the task from being stranded outside its allowed affinity. For the majority of cases, this would still keep select_fallback_rq() as O(N). cpumask_intersects_and(), which is O(N), is called only if !cpu_preferred. The task running there is expected to move out. Subsequently, it should run on a preferred CPU. This becomes O(N**2) only for tasks pinned solely to non-preferred CPUs. That is a rare case. Overhead is minimal when the CPU is preferred. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-7-sshegde@linux.ibm.com --- kernel/sched/core.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d4edee72d793..d095fb1be6e9 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2511,6 +2511,24 @@ static inline bool rq_has_pinned_tasks(struct rq *rq) return rq->nr_pinned; } +static inline bool task_can_migrate_to_preferred(struct task_struct *p, int cpu) +{ + /* No need to migrate from a preferred CPU */ + if (cpu_preferred(cpu)) + return false; + + /* Only FAIR tasks honor preferred CPU state */ + if (unlikely(p->sched_class != &fair_sched_class)) + return false; + + /* Ignore preferred state if task affinity is changing */ + if (unlikely(!cpumask_test_cpu(task_cpu(p), p->cpus_ptr))) + return false; + + return cpumask_intersects_and(p->cpus_ptr, cpu_preferred_mask, + task_cpu_possible_mask(p)); +} + /* * Per-CPU kthreads are allowed to run on !active && online CPUs, see * __set_cpus_allowed_ptr() and select_fallback_rq(). @@ -2526,8 +2544,12 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) return cpu_online(cpu); /* Non kernel threads are not allowed during either online or offline. */ - if (!(p->flags & PF_KTHREAD)) + if (!(p->flags & PF_KTHREAD)) { + /* Try to use preferred CPU if task's affinity allows */ + if (task_can_migrate_to_preferred(p, cpu)) + return false; return cpu_active(cpu); + } /* KTHREAD_IS_PER_CPU is always allowed. */ if (kthread_is_per_cpu(p)) @@ -2537,7 +2559,11 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) if (cpu_dying(cpu)) return false; - /* But are allowed during online. */ + /* Try to keep unbound kthreads on a preferred CPU if possible. */ + if (task_can_migrate_to_preferred(p, cpu)) + return false; + + /* Otherwise, they are allowed to run on online CPU. */ return cpu_online(cpu); } From 4ee29b029058a8f06fbb91425e1d9351e8015b53 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:22 +0530 Subject: [PATCH 40/64] sched/fair: Load balance only among preferred CPUs When a CPU is marked as non-preferred, any load pulled towards it is pointless since the task will be pushed out again in the next tick. So, consider only preferred CPUs for load balancing. This ensures load balancing does not fight against the push task mechanism which happens at the tick. Also, this stops active balancing from happening on a non-preferred CPU pulling the load. This also means there is no load balancing if a task is pinned only to non-preferred CPUs. They will continue to run where they were previously running before the CPUs were marked as non-preferred. Bail out early for NEWIDLE balancing, as load balancing is done only on preferred CPUs. Note that idle balancing is allowed to go through, since that naturally updates nohz.next_balance when all the idle CPUs are non-preferred. Also, optimization in find_new_ilb() is skipped. The steal governor driver, which is introduced in later patches, updates the preferred CPUs state in descending order. find_new_ilb() checks for idle CPUs in ascending order. Hence, in most common scenarios, the idle CPU found by find_new_ilb() will already be a preferred CPU. When all idle CPUs are non-preferred, the first idle CPU has to be chosen anyway. All of this is naturally handled in find_new_ilb() currently. Adding additional complexity to it for rare edge cases is not necessary. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yury Norov Link: https://patch.msgid.link/20260928053728.797539-8-sshegde@linux.ibm.com --- kernel/sched/fair.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 03206e15e6fe..1c687c3c70f1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -13467,7 +13467,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, }; bool need_unlock = false; - cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); + cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask); schedstat_inc(sd->lb_count[idle]); @@ -14582,10 +14582,8 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) */ this_rq->idle_stamp = rq_clock(this_rq); - /* - * Do not pull tasks towards !active CPUs... - */ - if (!cpu_active(this_cpu)) + /* Do not pull tasks towards !preferred CPUs */ + if (!cpu_preferred(this_cpu)) return 0; /* From 74699f56ebcfe0f401ee6ce4c5a72822101bb222 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:23 +0530 Subject: [PATCH 41/64] sched/core: Push current task from non preferred CPU Actively push out the current running task on a non-preferred CPU (NPC). Since the task is currently running, a stopper thread must be queued to push the task out. However, if the task is pinned only to non-preferred CPUs, it will continue running there. This helps to maintain userspace affinities, unlike CPU hotplug or isolated cpusets. The implementation follows the structure of __balance_push_cpu_stop(), but is kept separate to handle the preferred-CPU specific conditions and pending-work state under CONFIG_PREFERRED_CPU. Add the npc_push_work_pending flag to protect the work buffer. For now, only the currently running task is pushed out. This keeps the code simpler. In the future, an optimization may be added to move all queued tasks on the runqueue. This works only for the FAIR scheduling class. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-9-sshegde@linux.ibm.com --- kernel/sched/core.c | 83 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 9 +++++ 2 files changed, 92 insertions(+) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d095fb1be6e9..0e40055c2367 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5829,6 +5829,9 @@ void sched_tick(void) unsigned long hw_pressure; u64 resched_latency; + if (!cpu_preferred(cpu)) + sched_push_current_non_preferred_cpu(rq); + if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) arch_scale_freq_tick(); @@ -11253,3 +11256,83 @@ void sched_change_end(struct sched_change_ctx *ctx) p->sched_class->prio_changed(rq, p, ctx->prio); } } + +#ifdef CONFIG_PREFERRED_CPU +static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work); + +static int sched_non_preferred_cpu_push_stop(void *arg) +{ + struct task_struct *p = arg; + struct rq *rq = this_rq(); + struct rq_flags rf; + int cpu; + + if (cpu_preferred(rq->cpu)) { + scoped_guard(rq_lock_irqsave, rq) + rq->npc_push_work_pending = false; + put_task_struct(p); + return 0; + } + + scoped_guard (raw_spinlock_irq, &p->pi_lock) { + /* + * select_fallback_rq() may acquire the rq lock in case of + * fallback. So call it before grabbing rq lock. If the task + * migrates to another CPU before the rq lock is acquired, + * subsequent validation of task's current rq will help to + * safely bail out. + */ + cpu = select_fallback_rq(rq->cpu, p); + rq_lock(rq, &rf); + rq->npc_push_work_pending = false; + update_rq_clock(rq); + context_unsafe_alias(rq); + + if (task_rq(p) == rq && task_on_rq_queued(p)) + rq = __migrate_task(rq, &rf, p, cpu); + rq_unlock(rq, &rf); + } + + put_task_struct(p); + return 0; +} + +/* + * Push the current task running on non-preferred CPU(npc). + * Using this non preferred CPU will lead to more contention + * in the host. So it is better not to use this CPU. + * + * Since task is running, call a stopper to push the task out. This is + * similar to how task moves during hotplug. In select_fallback_rq() a + * preferred CPU will be chosen and henceforth task shouldn't come back to + * this CPU again. + * + * Works for FAIR class only. + * + * If task is affined only on non-preferred CPUs, no point in moving it out. + */ +void sched_push_current_non_preferred_cpu(struct rq *rq) +{ + struct task_struct *push_task = rq->curr; + + scoped_guard(rq_lock, rq) { + /* Push the task if its explicit affinity allows */ + if (!task_can_migrate_to_preferred(push_task, rq->cpu)) + return; + + /* There is already a stopper thread. Don't race with it. */ + if (rq->npc_push_work_pending) + return; + + if (is_migration_disabled(push_task)) + return; + + rq->npc_push_work_pending = true; + } + + /* sched_tick runs with interrupts disabled. */ + get_task_struct(push_task); + stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop, + push_task, this_cpu_ptr(&npc_push_task_work)); +} +#endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d41d23a5632b..7d2ec527b8a2 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1326,6 +1326,9 @@ struct rq { #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING u64 prev_steal_time_rq; #endif +#ifdef CONFIG_PREFERRED_CPU + bool npc_push_work_pending; +#endif /* calc_load related fields */ unsigned long calc_load_update; @@ -4299,4 +4302,10 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) #include "ext/ext.h" +#ifdef CONFIG_PREFERRED_CPU +void sched_push_current_non_preferred_cpu(struct rq *rq); +#else /* !CONFIG_PREFERRED_CPU */ +static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { } +#endif + #endif /* _KERNEL_SCHED_SCHED_H */ From 68957caaa9c0e127bc87fee64e45be843f6c5c91 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:24 +0530 Subject: [PATCH 42/64] sched/debug: Add migration stats due to non preferred CPUs Add a new per-task stat, - nr_migrations_cpu_non_preferred: number of push migrations while the CPU is non-preferred. Since this new stat is per-task, it changes only /proc//sched. It doesn't update /proc/schedstat. Hence increasing the schedstat version is not necessary. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-10-sshegde@linux.ibm.com --- include/linux/sched.h | 1 + kernel/sched/core.c | 9 +++++++-- kernel/sched/debug.c | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index d7cc77181ef9..48639b0c75d0 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -554,6 +554,7 @@ struct sched_statistics { u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; + u64 nr_migrations_cpu_non_preferred; u64 nr_wakeups; u64 nr_wakeups_sync; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0e40055c2367..0bb86a43a592 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -11288,8 +11288,13 @@ static int sched_non_preferred_cpu_push_stop(void *arg) update_rq_clock(rq); context_unsafe_alias(rq); - if (task_rq(p) == rq && task_on_rq_queued(p)) - rq = __migrate_task(rq, &rf, p, cpu); + if (task_rq(p) == rq && task_on_rq_queued(p)) { + struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu); + + if (rq != dest_rq) + schedstat_inc(p->stats.nr_migrations_cpu_non_preferred); + rq = dest_rq; + } rq_unlock(rq, &rf); } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fda10b3fa249..e6a3b516c703 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -1440,6 +1440,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); P_SCHEDSTAT(nr_forced_migrations); + P_SCHEDSTAT(nr_migrations_cpu_non_preferred); P_SCHEDSTAT(nr_wakeups); P_SCHEDSTAT(nr_wakeups_sync); P_SCHEDSTAT(nr_wakeups_migrate); From 9a8e740ee9f69ec857ffa0c76cf1a01d1cb7360f Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:25 +0530 Subject: [PATCH 43/64] virt: Introduce steal governor driver Introduce a new driver in virt named steal_governor. This driver will compute the steal time and drive the policy decisions regarding the preferred CPU state. Note that this driver is strictly intended for actual guests. Hence block it on Xen dom0. More details can be found in Documentation/driver-api/steal-governor.rst. A new kconfig called STEAL_GOVERNOR is introduced in subsequent patches, which enables this driver. This driver will select CONFIG_PREFERRED_CPU. This makes configs driven by user preference/configuration. When the driver is disabled, preferred CPUs remain the same as active CPUs. The file layout of the driver is kept simple for now. The code is in drivers/virt/steal_governor.c, and the configs are part of drivers/virt/Kconfig. The main structure of the steal governor contains: - work, delay: Deferred periodic work function variables. - steal, time: Used to calculate deltas during periodic work. - interval_ms, high_threshold, low_threshold: Tuning knobs for the steal governor. While there, add MAINTAINERS entry for this new driver. Suggested-by: Yury Norov Suggested-by: K Prateek Nayak Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-11-sshegde@linux.ibm.com --- Documentation/driver-api/index.rst | 1 + Documentation/driver-api/steal-governor.rst | 151 ++++++++++++++++++++ MAINTAINERS | 9 ++ drivers/virt/steal_governor.c | 78 ++++++++++ 4 files changed, 239 insertions(+) create mode 100644 Documentation/driver-api/steal-governor.rst create mode 100644 drivers/virt/steal_governor.c diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 6601a258690f..26b7638a327d 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -139,6 +139,7 @@ Subsystem-specific APIs sm501 soundwire/index spi + steal-governor surface_aggregator/index switchtec sync_file diff --git a/Documentation/driver-api/steal-governor.rst b/Documentation/driver-api/steal-governor.rst new file mode 100644 index 000000000000..3817eedb38d7 --- /dev/null +++ b/Documentation/driver-api/steal-governor.rst @@ -0,0 +1,151 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Steal Governor +============== + +:Author: Shrikanth Hegde + +Introduction +============ + +The steal governor is aimed at mitigating the Noisy Neighbour problem +which occurs in paravirtualized environments with CPU overcommit. +The performance of a workload running in one VM gets degraded by +the activity of other VMs on the same host. As a result, all VMs +collectively make slower forward progress. + +In such systems, high utilization in all VMs causes the hypervisor to +frequently preempt vCPUs. This vCPU preemption is expensive. +To mitigate this, the kernel aims to restrict workloads to a subset of +Preferred CPUs to reduce physical CPU contention. +A detailed explanation of Preferred CPUs is available in +``Documentation/scheduler/sched-paravirt.rst``. + +The steal governor selects ``CONFIG_PREFERRED_CPU=y`` which enables the +scheduler core infrastructure to move the tasks to Preferred CPUs where +possible. The driver controls the policy decisions regarding the state of +preferred CPUs. That is, this driver decides which CPUs are preferred +and which CPUs are non-preferred. + +The driver code is available at ``drivers/virt/steal_governor.c``. + +Core idea +========= + +steal time is an indication available today in Guest which shows contention +for underlying physical CPU. Use it as a hint in the guest to fold the +workload to a reduced set of vCPUs. When there is contention, steal time +will show up in all the guests. When each guest honors the hint and folds +the workload to a smaller set of vCPUs (Preferred CPUs), it reduces the +contention and thereby reduces vCPU preemption. +This is achieved without any cross-guest communication. + +Steal governor driver effectively does: + +1. Periodically computes the steal ratio using accumulated steal time + across possible CPUs, normalized by the number of active CPUs. + +2. If steal ratio is greater than high threshold, reduce the number of + preferred CPUs by 1 core. Ensure at least one core is left always. + Skip changing the state of offline CPUs in that core. + +3. If steal ratio is less than or equal to low threshold, increase the + number of preferred CPUs by 1 core. If preferred is same as active, + nothing to be done. Skip changing the state of offline CPUs. + This helps to handle cases where few CPUs are offline in a core and + those offline CPUs will not be marked as preferred. + +4. Ensure preferred CPUs is always subset of active CPUs. + On feature disable it is same as active CPUs. + +This feature works best only when all the VMs enable the feature as +it is a co-operative scheme. If a specific VM doesn't enable this feature +it may end up with more CPUs than others, still should lead to better +performance when seen from system view. Those who enable this driver must +ensure it is enabled in all VMs. + +Note that this driver is strictly intended for actual guests; for example, +loading this module in a privileged VM like Xen Dom0 is blocked. + +Workload considerations +======================= + +The steal governor is useful for workloads where vCPU preemption has +costs beyond the lost CPU time, such as lock-holder preemption, critical +sections, communicating threads, and cache or TLB disruption. + +Pure CPU-time workloads with independent workers may not benefit and +could see a small regression due to additional guest scheduling overhead. + +Module Parameters +================= + +interval_ms +----------- + +How often steal governor checks for steal time. +Default: 1000 i.e. 1 second. Value should be in between 100ms to 100sec. + +This controls how fast steal governor driver reacts to changes to the +contention of physical CPUs. Since it does a fair amount of work, setting +too low may have overhead. Setting it too high might render it ineffective. + +low_threshold +------------- + +lower threshold value in percentage * 100. +Default: 200, i.e. 2% steal is considered as low threshold. +Can't be higher than high_threshold. + +This determines what values should be considered as nil/no steal values. +When steal governor sees steal ratio is less than or equal to this value, +it will increase the preferred CPUs by 1 core. +Using zero might cause oscillations. + +high_threshold +-------------- + +higher threshold value in percentage * 100 +Default: 500, i.e. 5% steal is considered as high threshold. +Can't be lower than low_threshold. Must be less than 10000. + +This determines what values should be considered as high steal values. +When steal governor sees steal ratio is higher than this value, it will +reduce the preferred CPUs by 1 core. + +Limitations of default values +----------------------------- + +Because of the vast diversity in VM configurations and different +architectures, the default thresholds may not be optimal for all systems. +Users may need to tune these parameters based on the system under +test to achieve the best results. + +The governor sums the steal time across all possible CPUs, which ensures +the accumulated steal time remains a monotonically increasing value. +However, to calculate the effective steal ratio, it divides this sum +by the number of active CPUs. Because only active CPUs contribute to +the steal time delta, this prevents threshold dilution on sparsely +populated systems. + +The driver reduces/increases preferred CPUs by core-level. This could provide +faster convergence for hypervisors such as powerVM. But on KVM and Xen +convergence could be slower depending on the configuration. +Using a smaller interval_ms could help one to expedite it. + +Reasons for CONFIG_STEAL_GOVERNOR=m +=================================== + +Selecting this driver makes CONFIG_PREFERRED_CPU=y. That makes configs +driven by user preference. Though one can have CONFIG_STEAL_GOVERNOR=y, +It is recommended to build CONFIG_STEAL_GOVERNOR=m due to below reasons: + +1. Doing periodic work has additional overheads. Enabling this driver + in systems where steal time cannot happen is of no use. There is no + benefit with additional overheads in such systems. + +2. This works well when all VMs work in a co-operative manner. When an + administrative user enables it in one VM, he/she will likely enable + it in all VMs. + +3. User can tweak the module parameters by reloading the module. diff --git a/MAINTAINERS b/MAINTAINERS index 3a19da74d00c..28003d36b3f7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -26223,6 +26223,15 @@ F: rust/helpers/jump_label.c F: rust/kernel/generated_arch_static_branch_asm.rs.S F: rust/kernel/jump_label.rs +STEAL GOVERNOR DRIVER +M: Shrikanth Hegde +R: Yury Norov +L: linux-kernel@vger.kernel.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core +F: Documentation/driver-api/steal-governor.rst +F: drivers/virt/steal_governor.c + STI AUDIO (ASoC) DRIVERS M: Arnaud Pouliquen L: linux-sound@vger.kernel.org diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c new file mode 100644 index 000000000000..2320cbfa4b47 --- /dev/null +++ b/drivers/virt/steal_governor.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Steal time governor driver periodically computes steal time. + * Based on the thresholds it either reduce/increase the preferred + * CPUs which can be used by the workload to avoid vCPU preemption + * to an extent possible in paravirtualized environment. + * + * Available with CONFIG_STEAL_GOVERNOR + * + * Copyright (C) 2026 IBM + * Author: Shrikanth Hegde + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_XEN +#include +#endif + +#if !IS_ENABLED(CONFIG_PREFERRED_CPU) +#error "Steal Governor requires CONFIG_PREFERRED_CPU" +#endif + +struct steal_governor { + ktime_t time; + u64 steal; + unsigned long delay; + unsigned int interval_ms; + unsigned int high_threshold; + unsigned int low_threshold; + struct delayed_work work; +}; + +static struct steal_governor sg_ctx; + +static void restore_preferred_to_active(void) +{ + int cpu; + + guard(cpus_read_lock)(); + for_each_cpu(cpu, cpu_active_mask) + set_cpu_preferred(cpu, true); +} + +static int __init steal_governor_init(void) +{ +#ifdef CONFIG_XEN + if (xen_initial_domain()) { + pr_err("Cannot load in Xen Dom0 (Host OS). Driver is for guests only.\n"); + return -ENODEV; + } +#endif + + pr_info("enabled\n"); + return 0; +} + +static void __exit steal_governor_exit(void) +{ + restore_preferred_to_active(); + pr_info("disabled\n"); +} + +module_init(steal_governor_init); +module_exit(steal_governor_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("Virtualization Steal Time Governor"); From 4b9302d494fffee7318bd9cdd2021198b6a7badd Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:26 +0530 Subject: [PATCH 44/64] virt/steal_governor: Add control knobs for handling steal values These are the knobs to control the steal_governor. interval_ms: How often steal governor checks for steal time. (Default: 1000 i.e 1 second) This controls how fast steal governor driver reacts to changes to the contention of physical CPUs. Can be set between 100 to 100000. i.e. 100ms to 100seconds. 100ms is kept as minimum to ensure few meaningful steal values accumulate even with HZ=100. low_threshold: lower threshold value in percentage * 100. (Default: 200, i.e 2% steal is considered as low threshold) This determines what values should be considered as nil/no steal values. When steal governor see steal ratio is below or equal to this value, it will increase the preferred CPUs by 1 core. Having value as zero might cause oscillations high_threshold: higher threshold value in percentage * 100 (Default: 500, i.e 5% steal is considered as high threshold) This determines what values should be considered as high steal values. When steal governor sees steal ratio is higher than this value, it will reduce the preferred CPUs by 1 core. module_param_cb methods are used to do the validation checks. This helps to ensure one configures sane values. Since low and high are dependent, that check is done at module init. Notes: - Parameters values can't be changed at runtime. One has to unload the module and change it. Hence recommended to build it as module. - Default values may not work well for all configurations. Tune it according to the system under test. Documentation is available at: Documentation/driver-api/steal-governor.rst Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-12-sshegde@linux.ibm.com --- drivers/virt/steal_governor.c | 73 ++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c index 2320cbfa4b47..27f53ea16498 100644 --- a/drivers/virt/steal_governor.c +++ b/drivers/virt/steal_governor.c @@ -40,7 +40,11 @@ struct steal_governor { struct delayed_work work; }; -static struct steal_governor sg_ctx; +static struct steal_governor sg_ctx = { + .interval_ms = 1000, /* 1 second */ + .high_threshold = 500, /* 5% */ + .low_threshold = 200, /* 2% */ +}; static void restore_preferred_to_active(void) { @@ -51,6 +55,62 @@ static void restore_preferred_to_active(void) set_cpu_preferred(cpu, true); } +static int param_set_interval_ms(const char *val, const struct kernel_param *kp) +{ + unsigned int interval; + int ret; + + ret = kstrtouint(val, 0, &interval); + if (ret) + return ret; + + if (interval < 100 || interval > 100000) { + pr_err("interval_ms must be between 100 and 100000\n"); + return -EINVAL; + } + + return param_set_uint(val, kp); +} + +static const struct kernel_param_ops interval_ms_ops = { + .set = param_set_interval_ms, + .get = param_get_uint, +}; + +module_param_cb(interval_ms, &interval_ms_ops, &sg_ctx.interval_ms, 0444); +MODULE_PARM_DESC(interval_ms, + "Sampling frequency in milliseconds. default: 1000"); + +static int param_set_high_threshold(const char *val, const struct kernel_param *kp) +{ + unsigned int threshold; + int ret; + + ret = kstrtouint(val, 0, &threshold); + if (ret) + return ret; + + if (threshold >= 100 * 100) { + pr_err("high_threshold (%u) can't be more than 99.99%%\n", threshold); + return -EINVAL; + } + + return param_set_uint(val, kp); +} + +static const struct kernel_param_ops high_threshold_ops = { + .set = param_set_high_threshold, + .get = param_get_uint, +}; + +module_param_cb(high_threshold, &high_threshold_ops, &sg_ctx.high_threshold, 0444); +MODULE_PARM_DESC(high_threshold, + "High steal threshold. default: 500 i.e 5%. Must be > low_threshold"); + +module_param_named(low_threshold, sg_ctx.low_threshold, uint, 0444); +MODULE_PARM_DESC(low_threshold, + "Low steal threshold. default: 200 i.e 2%. Must be < high_threshold"); + static int __init steal_governor_init(void) { #ifdef CONFIG_XEN @@ -60,7 +120,16 @@ static int __init steal_governor_init(void) } #endif - pr_info("enabled\n"); + if (sg_ctx.low_threshold >= sg_ctx.high_threshold) { + pr_err("low_threshold (%u) must be less than high_threshold (%u)\n", + sg_ctx.low_threshold, sg_ctx.high_threshold); + return -EINVAL; + } + + sg_ctx.delay = msecs_to_jiffies(sg_ctx.interval_ms); + pr_info("enabled. interval: %ums, high_threshold: %u, low_threshold: %u\n", + sg_ctx.interval_ms, sg_ctx.high_threshold, sg_ctx.low_threshold); + return 0; } From 27d47ebce4d6490ce093031a48cc2d81bf9997a3 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:27 +0530 Subject: [PATCH 45/64] virt/steal_governor: Implement steal_governor policy loop Schedule work at regular intervals to implement the steal_governor policy loop, which monitors steal time and takes action on the state of preferred CPUs. The interval is determined by the interval_ms parameter. schedule_delayed_work() is used since interval_ms is on the order of milliseconds and the work does not need to happen instantly. Periodic policy loop essentially does: - Gets the total/delta steal values and cpus to use steal_ratio. - Calculate the steal_ratio as below. steal_ratio = (delta_steal * 100*100)/(delta_ns * num_cpus()) It is calculated this way to consider the fractional values of steal time. I.e 10 means 0.1% steal time. A few tricks such as divide by 10,000 are used to avoid possible overflow. - If steal ratio is higher than high threshold, call the method to reduce the preferred CPUs. - If steal ratio is lower or equal to low threshold, call the method to increase the preferred CPUs. - If the steal ratio falls in between, no action is taken. - Ensures design constraints always met. 1. At least one core/CPU must be there in preferred mask. 2. preferred CPUs is subset of active CPUs. If not met, then restore preferred CPUs to active and stop requeue of the work. Driver is effectively non-functional after that. Note that design checks are always performed. This helps avoid placing driver-specific design constraints inside the core CPU hotplug mechanism. User may offline specific set of CPUs that could leave the preferred mask as empty. With the design check performed always, driver gracefully shuts down upon detecting that edge case. In order to help the above loop, a few helper functions have been added. 1. get_system_steal_time() - steal governor takes global view of steal time instead of individual vCPU. Collect the steal values across the vCPUs of interest. - Sum up steal time values across possible CPUs. This helps to keep it a monotonically increasing number and avoids spikes due to CPU hotplug. 2. decrease_preferred_cpus() - Called when there is high steal time. It needs to decide which CPUs to mark as non-preferred. - Get first housekeeping CPU and its core mask. Mark it as protected core. This helps to keep at least one core as preferred. (kernel ensures at least one housekeeping CPU stays active.) - Find the last CPU outside of this protected core mask. i.e target CPU - Based on that target CPU, get its sibling and mark them as non-preferred. 3. increase_preferred_cpus() - Called when there is low steal time. It needs to decide which CPUs to mark as preferred and set that state. - Get the first active non-preferred CPUs. This likely is the last set of CPUs being marked as non-preferred. - get the siblings of that CPU and mark them as preferred. 4. get_system_cpus() - informs how many CPUs needs to be considered for steal_ratio calculations. - Since only active CPUs effectively contribute to steal time delta, returns number of active CPUs. This also helps to avoid dilution of thresholds in sparsely populated systems. Notes: 1. Using core instead of individual CPUs performs better as SMT is quite common and some hypervisor such as powerVM does core scheduling. 2. This doesn't do any NUMA splicing to keep the code simpler and minimal overhead. Current code expects CPUs spread uniformly across NUMA nodes. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-13-sshegde@linux.ibm.com --- drivers/virt/steal_governor.c | 149 ++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c index 27f53ea16498..6e31f9923dea 100644 --- a/drivers/virt/steal_governor.c +++ b/drivers/virt/steal_governor.c @@ -13,13 +13,18 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include #include +#include #include #include +#include #include +#include +#include #include #include #ifdef CONFIG_XEN @@ -111,6 +116,145 @@ module_param_named(low_threshold, sg_ctx.low_threshold, uint, 0444); MODULE_PARM_DESC(low_threshold, "Low steal threshold. default: 200 i.e 2%. Must be < high_threshold"); +/* Return collective steal time across system. */ +static u64 get_system_steal_time(void) +{ + return kcpustat_field_total(CPUTIME_STEAL, cpu_possible_mask); +} + +/* Return number of CPUs to consider for steal ratio. */ +static unsigned int get_system_cpus(void) +{ + return num_active_cpus(); +} + +/* + * Called when the steal governor detects high physical CPU contention. + * It finds the last active core in the preferred mask and mark those + * CPUs as non-preferred. + * + * Must ensure: + * - at least one core is always kept as preferred + * - preferred is always subset of active. + */ +static void decrease_preferred_cpus(void) +{ + const struct cpumask *first_hk_core; + int target_cpu = nr_cpu_ids; + int cpu; + + guard(cpus_read_lock)(); + cpu = cpumask_first_and(housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), + cpu_preferred_mask); + if (cpu >= nr_cpu_ids) + return; + + /* Always leave first housekeeping core as preferred. */ + first_hk_core = topology_sibling_cpumask(cpu); + cpu = cpumask_last(cpu_preferred_mask); + if (cpu >= nr_cpu_ids) + return; + + /* Find the last CPU which doesn't belong to that first hk_core. */ + if (!cpumask_test_cpu(cpu, first_hk_core)) { + target_cpu = cpu; + } else { + for_each_cpu_andnot(cpu, cpu_preferred_mask, first_hk_core) + target_cpu = cpu; + } + + /* Only the first housekeeping core remains */ + if (target_cpu >= nr_cpu_ids) + return; + + for_each_cpu_and(cpu, topology_sibling_cpumask(target_cpu), + cpu_preferred_mask) + set_cpu_preferred(cpu, false); +} + +/* + * Called when the steal governor detects no/low physical CPU contention. + * It finds the first active core outside of preferred mask and mark + * those CPUs as preferred. + * + * Must ensure preferred is subset of active. + */ +static void increase_preferred_cpus(void) +{ + int first_cpu, cpu; + + guard(cpus_read_lock)(); + first_cpu = cpumask_first_andnot(cpu_active_mask, cpu_preferred_mask); + + /* All CPUs are preferred. Nothing to increase further */ + if (first_cpu >= nr_cpu_ids) + return; + + for_each_cpu_and(cpu, topology_sibling_cpumask(first_cpu), + cpu_active_mask) + set_cpu_preferred(cpu, true); +} + +static bool preferred_cpus_valid(void) +{ + if (cpumask_empty(cpu_preferred_mask)) { + pr_err("empty preferred mask. stopping\n"); + return false; + } + + if (!cpumask_subset(cpu_preferred_mask, cpu_active_mask)) { + pr_err("preferred: %*pbl is not subset of active: %*pbl, stopping\n", + cpumask_pr_args(cpu_preferred_mask), + cpumask_pr_args(cpu_active_mask)); + return false; + } + + return true; +} + +static void steal_governor_loop(struct work_struct *work) +{ + u64 curr_steal, delta_steal, delta_ns, steal_ratio; + ktime_t now; + + now = ktime_get(); + delta_ns = ktime_to_ns(ktime_sub(now, sg_ctx.time)); + + if (unlikely(delta_ns < NSEC_PER_MSEC)) { + pr_err_ratelimited("work scheduled too soon delta_ns: %llu\n", delta_ns); + goto requeue_work; + } + + curr_steal = get_system_steal_time(); + delta_steal = curr_steal > sg_ctx.steal ? curr_steal - sg_ctx.steal : 0; + sg_ctx.steal = curr_steal; + sg_ctx.time = now; + + /* + * steal_ratio = (delta_steal * 100*100)/(delta_ns * num_cpus()) + * To avoid possible overflow, divide the denominator early. + * Note minimum interval is 100ms. + */ + delta_ns = max_t(u64, div_u64(delta_ns * get_system_cpus(), 10000), 1); + steal_ratio = div64_u64(delta_steal, delta_ns); + + if (steal_ratio > sg_ctx.high_threshold) + decrease_preferred_cpus(); + else if (steal_ratio <= sg_ctx.low_threshold) + increase_preferred_cpus(); + /* + * else: steal ratio is within bounds. Still do design checks so that + * module restores to active if CPU hotplug breaks those assumptions. + */ + if (!preferred_cpus_valid()) { + restore_preferred_to_active(); + return; + } + +requeue_work: + schedule_delayed_work(&sg_ctx.work, sg_ctx.delay); +} + static int __init steal_governor_init(void) { #ifdef CONFIG_XEN @@ -127,6 +271,10 @@ static int __init steal_governor_init(void) } sg_ctx.delay = msecs_to_jiffies(sg_ctx.interval_ms); + INIT_DELAYED_WORK(&sg_ctx.work, steal_governor_loop); + sg_ctx.steal = get_system_steal_time(); + sg_ctx.time = ktime_get(); + schedule_delayed_work(&sg_ctx.work, sg_ctx.delay); pr_info("enabled. interval: %ums, high_threshold: %u, low_threshold: %u\n", sg_ctx.interval_ms, sg_ctx.high_threshold, sg_ctx.low_threshold); @@ -135,6 +283,7 @@ static int __init steal_governor_init(void) static void __exit steal_governor_exit(void) { + disable_delayed_work_sync(&sg_ctx.work); restore_preferred_to_active(); pr_info("disabled\n"); } From 1fb28c664a19df8d45a6afa04d28d102b04ea680 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:28 +0530 Subject: [PATCH 46/64] virt/steal_governor: Enable the driver Provide a config option to enable the steal_governor driver. Since the feature targets paravirtualized environments and requires SMP, enforce those dependencies. The driver selects CONFIG_PREFERRED_CPU for the core scheduler mechanisms to work. It is recommended to build the driver as a module (m) instead of built-in (y) due to the following reasons: - Module parameters are read-only after initialization. Building as a module allows updating these parameters by simply reloading the module. Default module parameters cannot work in all configurations. - The driver can be completely disabled by unloading the module. - This feature works best when all VMs operate in a cooperative manner. Requiring an explicit module load ensures intentional deployment across all VMs by the system administrator. Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-14-sshegde@linux.ibm.com --- drivers/virt/Kconfig | 17 +++++++++++++++++ drivers/virt/Makefile | 1 + 2 files changed, 18 insertions(+) diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index 52eb7e4ba71f..eeb84e578ddf 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -41,6 +41,23 @@ config FSL_HV_MANAGER 4) A kernel interface for receiving callbacks when a managed partition shuts down. +config STEAL_GOVERNOR + tristate "Dynamic vCPU management based on steal time" + depends on PARAVIRT && SMP + select PREFERRED_CPU + default m + help + This driver helps to reduce the steal time in paravirtualized + environments, thereby reducing vCPU preemption costs. + + By default preferred CPUs will be same as active CPUs. Depending + on the steal time when steal_governor driver is enabled, + preferred CPUs could become subset of active CPUs. + More details are at: Documentation/driver-api/steal-governor.rst + + It is recommended to build it as module and load the module + to enable it. + source "drivers/virt/vboxguest/Kconfig" source "drivers/virt/nitro_enclaves/Kconfig" diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile index f29901bd7820..05fb075ef5b8 100644 --- a/drivers/virt/Makefile +++ b/drivers/virt/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_FSL_HV_MANAGER) += fsl_hypervisor.o obj-$(CONFIG_VMGENID) += vmgenid.o +obj-$(CONFIG_STEAL_GOVERNOR) += steal_governor.o obj-y += vboxguest/ obj-$(CONFIG_NITRO_ENCLAVES) += nitro_enclaves/ From 4b1f75be23c4fb0016a102d1cb108ba355c4c00d Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Tue, 29 Sep 2026 22:17:12 +0530 Subject: [PATCH 47/64] sched/core: Fix context analysis errors in non-preferred CPU push Nathan reported build failure with clang23 when CONFIG_PREFERRED_CPU=y. kernel/sched/core.c:11298:3: error: releasing raw_spinlock 'rq_lockp(rq)' that was not held [-Werror,-Wthread-safety-analysis] 11298 | rq_unlock(rq, &rf); | ^ kernel/sched/core.c:11303:1: error: raw_spinlock 'rq_lockp(__this_rq())' is not held on every path through here [-Werror,-Wthread-safety-analysis] context_unsafe_alias(rq) was placed after rq_lock(), by which point it had already associated the lock with the original rq alias. Move the alias barrier before rq_lock() so that the changing rq alias is accounted for before acquiring the lock. Fixes: 74699f56ebcf ("sched/core: Push current task from non preferred CPU") Closes: https://lore.kernel.org/all/20260929121838.GA1814129@ax162/ Reported-by: Nathan Chancellor Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Tested-by: Nathan Chancellor # build Link: https://patch.msgid.link/20260929164712.1054883-1-sshegde@linux.ibm.com --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0bb86a43a592..23677d76f9d2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -11283,10 +11283,10 @@ static int sched_non_preferred_cpu_push_stop(void *arg) * safely bail out. */ cpu = select_fallback_rq(rq->cpu, p); + context_unsafe_alias(rq); rq_lock(rq, &rf); rq->npc_push_work_pending = false; update_rq_clock(rq); - context_unsafe_alias(rq); if (task_rq(p) == rq && task_on_rq_queued(p)) { struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu); From 53bc5c556b82a3ca32b62ba349c4511f91526b9f Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Tue, 30 Jun 2026 08:47:37 +0000 Subject: [PATCH 48/64] sched: Set TIF_NEED_RESCHED before calling __trace_set_need_resched() set_tsk_need_resched() tests TIF_NEED_RESCHED, calls __trace_set_need_resched() if the flag is clear, then sets it via set_tsk_thread_flag(). A BPF raw_tp program attached to sched_set_need_resched executes synchronously inside __bpf_trace_run(). On return, __bpf_trace_run() drops the RCU lock with rcu_read_unlock_migrate(), which on the preempt-or-BH-disabled path calls set_need_resched_current() -> set_tsk_need_resched() again. set_tsk_thread_flag() follows the tracepoint call, so every re-entrant frame sees TIF_NEED_RESCHED clear and calls __trace_set_need_resched() again: BUG: TASK stack guard page was hit at ffffc9001224ff98 Oops: stack guard page: 0000 [#1] SMP KASAN PTI RIP: 0010:__bpf_trace_sched_set_need_resched_tp+0x1c/0x190 Call Trace: trace_sched_set_need_resched_tp+0x110/0x130 set_tsk_need_resched include/linux/sched.h:2076 set_need_resched_current include/linux/sched.h:2094 rcu_read_unlock_special+0x43a/0x440 __rcu_read_unlock+0x9e/0x120 rcu_read_unlock_migrate+0xa9/0x240 __bpf_trace_run+0x131/0x180 bpf_trace_run3+0x333/0x430 __bpf_trace_sched_set_need_resched_tp+0x13a/0x190 trace_sched_set_need_resched_tp+0x110/0x130 set_tsk_need_resched include/linux/sched.h:2076 ... __resched_curr() has the same ordering, firing the tracepoint before setting the flag via set_ti_thread_flag() or set_nr_and_not_polling(). Fix it for consistency. Replace the separate test_tsk_thread_flag() + set_tsk_thread_flag() pair in set_tsk_need_resched() with test_and_set_tsk_thread_flag(). In __resched_curr(), move the tracepoint call after the flag is set in each path. Fixes: adcc3bfa8806 ("sched: Adapt sched tracepoints for RV task model") Signed-off-by: Sechang Lim Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andrea Righi Acked-by: Gabriele Monaco Link: https://patch.msgid.link/20260630084750.2792851-1-rhkrqnwk98@gmail.com --- include/linux/sched.h | 5 ++--- kernel/sched/core.c | 7 +++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 48639b0c75d0..7b91cae4a59c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2105,10 +2105,9 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) static inline void set_tsk_need_resched(struct task_struct *tsk) { - if (tracepoint_enabled(sched_set_need_resched_tp) && - !test_tsk_thread_flag(tsk, TIF_NEED_RESCHED)) + if (!test_and_set_tsk_thread_flag(tsk, TIF_NEED_RESCHED) && + tracepoint_enabled(sched_set_need_resched_tp)) __trace_set_need_resched(tsk, TIF_NEED_RESCHED); - set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); } static inline void clear_tsk_need_resched(struct task_struct *tsk) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 23677d76f9d2..71d3c948e14a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1196,6 +1196,7 @@ static void __resched_curr(struct rq *rq, int tif) { struct task_struct *curr = rq->curr; struct thread_info *cti = task_thread_info(curr); + bool need_ipi; int cpu; lockdep_assert_rq_held(rq); @@ -1212,15 +1213,17 @@ static void __resched_curr(struct rq *rq, int tif) cpu = cpu_of(rq); - trace_sched_set_need_resched_tp(curr, cpu, tif); if (cpu == smp_processor_id()) { set_ti_thread_flag(cti, tif); if (tif == TIF_NEED_RESCHED) set_preempt_need_resched(); + trace_sched_set_need_resched_tp(curr, cpu, tif); return; } - if (set_nr_and_not_polling(cti, tif)) { + need_ipi = set_nr_and_not_polling(cti, tif); + trace_sched_set_need_resched_tp(curr, cpu, tif); + if (need_ipi) { if (tif == TIF_NEED_RESCHED) smp_send_reschedule(cpu); } else { From c8fc4136fd3c58458592b77168f0916d08dbea5a Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 29 Sep 2026 18:54:40 +0200 Subject: [PATCH 49/64] sched/fair: Honor asymmetric SMT priority in idle selection POWER7 uses SD_ASYM_PACKING at the shared-capacity SMT level to order hardware threads, and NVIDIA Olympus benefits from the same policy. Idle CPU selection does not consult that order, so a task can wake on an arbitrary sibling and remain there until load balancing corrects the placement. On these systems, that initial choice can prevent the core from entering its preferred lower-thread resource mode and cause a large and persistent performance loss. When idle selection finds an available CPU in an SMT core, choose the highest-priority available sibling. On SMT2 Olympus this only changes selection on fully idle cores. A partially idle core has only one available CPU. On wider SMT systems such as POWER7, it also fills available siblings in priority order while the core is partially busy. Apply the preference to idle-core and idle-CPU scans, asymmetric-capacity scans, target, previous, recently-used CPU fast paths and the slow path. Inspect the lowest scheduling domain directly, but require both CPUs to share its span because isolcpus can split hardware siblings across scheduling domains. Keep physical-core capacity selection independent from SMT sibling ordering. SD_ASYM_CPUCAPACITY first selects among cores with different maximum capacities, then SD_ASYM_PACKING selects the preferred available sibling inside the chosen core, whose siblings continue to share equal capacity. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Srikar Dronamraju Reviewed-by: K Prateek Nayak Reviewed-by: Vincent Guittot Reviewed-by: Kayra Cizmeci Tested-by: K Prateek Nayak Tested-by: Kayra Cizmeci Tested-by: Breno Leitao Link: https://patch.msgid.link/20260929165538.726616-2-arighi@nvidia.com --- kernel/sched/fair.c | 85 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1c687c3c70f1..b42a0ca2a69e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8575,6 +8575,35 @@ static inline bool test_idle_cores(int cpu) return false; } +/* + * Redirect a CPU to a higher-priority available sibling in its SMT domain, + * subject to task affinity. + */ +static inline int select_idle_smt_cpu(struct task_struct *p, int cpu) +{ + struct sched_domain *sd; + int best = cpu; + int sibling; + + if (!sched_smt_active()) + return cpu; + + sd = rcu_dereference_all(cpu_rq(cpu)->sd); + if (!sd || !(sd->flags & SD_SHARE_CPUCAPACITY) || + !(sd->flags & SD_ASYM_PACKING)) + return cpu; + + for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) { + if (sibling == best || !choose_idle_cpu(sibling, p)) + continue; + + if (sched_asym_prefer(sibling, best)) + best = sibling; + } + + return best; +} + /* * Scans the local SMT mask to see if the entire core is idle, and records this * information in sd_balance_shared->has_idle_cores. @@ -8959,7 +8988,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (choose_idle_cpu(target, p) && asym_fits_cpu(task_util, util_min, util_max, target)) - return target; + goto select_smt_priority; /* * If the previous CPU is cache affine and idle, don't be stupid: @@ -8969,8 +8998,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, prev)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(prev, target)) - return prev; + cpus_share_resources(prev, target)) { + target = prev; + goto select_smt_priority; + } prev_aff = prev; } @@ -8988,7 +9019,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) prev == smp_processor_id() && this_rq()->nr_running <= 1 && asym_fits_cpu(task_util, util_min, util_max, prev)) { - return prev; + target = prev; + goto select_smt_priority; } /* Check a recently used CPU as a potential idle candidate: */ @@ -9002,8 +9034,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(recent_used_cpu, target)) - return recent_used_cpu; + cpus_share_resources(recent_used_cpu, target)) { + target = recent_used_cpu; + goto select_smt_priority; + } } else { recent_used_cpu = -1; @@ -9025,7 +9059,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (sd) { i = select_idle_capacity(p, sd, target); - return ((unsigned)i < nr_cpumask_bits) ? i : target; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } + return target; } } @@ -9038,14 +9076,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (!has_idle_core && cpus_share_cache(prev, target)) { i = select_idle_smt(p, sd, prev); - if ((unsigned int)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } } } i = select_idle_cpu(p, sd, has_idle_core, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } /* * For cluster machines which have lower sharing cache like L2 or @@ -9053,12 +9095,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * first. But prev_cpu or recent_used_cpu may also be a good candidate, * use them if possible when no idle CPU found in select_idle_cpu(). */ - if ((unsigned int)prev_aff < nr_cpumask_bits) - return prev_aff; - if ((unsigned int)recent_used_cpu < nr_cpumask_bits) - return recent_used_cpu; + if ((unsigned int)prev_aff < nr_cpumask_bits) { + target = prev_aff; + goto select_smt_priority; + } + if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { + target = recent_used_cpu; + goto select_smt_priority; + } return target; + +select_smt_priority: + return select_idle_smt_cpu(p, target); } /** @@ -9735,8 +9784,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) } /* Slow path */ - if (unlikely(sd)) - return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + if (unlikely(sd)) { + new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + return select_idle_smt_cpu(p, new_cpu); + } /* Fast path */ if (wake_flags & WF_TTWU) From 648d44bda7314ffb5805f2bea8910b4cdccaecd0 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 29 Sep 2026 18:54:41 +0200 Subject: [PATCH 50/64] sched/topology: Add asymmetric SMT packing override Architectures can use SD_ASYM_PACKING to describe preferred CPU ordering at the SMT scheduling domain. Some systems benefit from this policy, but their firmware cannot describe the preference and inferring it from the CPU model would embed a platform-specific policy in the kernel. Add sched_smt_asym_packing=on boot option to force SD_ASYM_PACKING at the SMT level. Omitting the option preserves the topology provided by the architecture or firmware. Apply the override centrally to domains with SD_SHARE_CPUCAPACITY so it also covers architectures with custom SMT topology callbacks, including powerpc. When enabled, priority remains defined by arch_asym_cpu_priority(). The weak default prefers lower-numbered logical CPUs, while architecture overrides remain authoritative. Siblings with equal priorities remain unordered. In particular, x86 with CONFIG_SCHED_MC_PRIO normally gives both SMT siblings the same core priority, so enabling the option there does not prioritize a sibling. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Tested-by: Breno Leitao Link: https://patch.msgid.link/20260929165538.726616-3-arighi@nvidia.com --- .../admin-guide/kernel-parameters.txt | 12 ++++++++++++ kernel/sched/topology.c | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 68647ff4bdd2..8f53d66afbf9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6836,6 +6836,18 @@ Kernel parameters solution to mutex-based priority inversion. Format: + sched_smt_asym_packing= [KNL,SMP] + Format: on + Force asymmetric packing at the SMT scheduling domain. + Idle CPU selection prefers siblings with a higher + arch_asym_cpu_priority(). The default implementation + prefers lower-numbered logical CPUs, but architecture + overrides remain authoritative. Equal priorities do + not establish a sibling preference. For example, x86 + with CONFIG_SCHED_MC_PRIO normally assigns the same + core priority to both SMT siblings, so this option + does not favor either sibling. + sched_verbose [KNL,EARLY] Enables verbose scheduler debug messages. schedstats= [KNL,X86] Enable or disable scheduled statistics. diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0248227d983a..dd2b1d70ec9d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -32,6 +32,20 @@ static int __init sched_debug_setup(char *str) } early_param("sched_verbose", sched_debug_setup); +#ifdef CONFIG_SCHED_SMT +static bool sched_smt_asym_packing __read_mostly; + +static int __init setup_sched_smt_asym_packing(char *str) +{ + if (strcmp(str, "on")) + return 0; + + sched_smt_asym_packing = true; + return 1; +} +__setup("sched_smt_asym_packing=", setup_sched_smt_asym_packing); +#endif + static inline bool sched_debug(void) { return sched_debug_verbose; @@ -1950,6 +1964,10 @@ sd_init(struct sched_domain_topology_level *tl, if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, "wrong sd_flags in topology description\n")) sd_flags &= TOPOLOGY_SD_FLAGS; +#ifdef CONFIG_SCHED_SMT + if (sched_smt_asym_packing && (sd_flags & SD_SHARE_CPUCAPACITY)) + sd_flags |= SD_ASYM_PACKING; +#endif sd_flags |= asym_cpu_capacity_classify(sd_span, cpu_map); *sd = (struct sched_domain){ From 791b1760accdc15248929f9b767410a76be199b1 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 16:38:12 +0200 Subject: [PATCH 51/64] irq_work: Update a comment regarding CPU hotplug invocation hotplug_cfd() no longer exists and the flow changed a bit since the rework of CPU hotplug as of commit 31487f8328f20 ("smp/cfd: Convert core to hotplug state machine"). Update the comment to reflect reality as of today. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911143815.997254-2-bigeasy@linutronix.de --- kernel/irq_work.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kernel/irq_work.c b/kernel/irq_work.c index f7e2dc2c30c6..9f19c05c9962 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -252,10 +252,7 @@ static void irq_work_run_list(struct llist_head *list) irq_work_single(work); } -/* - * hotplug calls this through: - * hotplug_cfd() -> flush_smp_call_function_queue() - */ +/* CPU hotplug calls this through smpcfd_dying_cpu() */ void irq_work_run(void) { irq_work_run_list(this_cpu_ptr(&raised_list)); From 40dcc9bdbef3d93a516c8ee32cb1b50eddeeebcc Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 16:38:13 +0200 Subject: [PATCH 52/64] irq_work: Flush lazy work CPU down on PREEMPT_RT PREEMPT_RT invokes IRQ_WORK_LAZY callbacks and callbacks which are not explicitly marked IRQ_WORK_HARD_IRQ from thread context. If a CPU gets shutdown, the irq_work is flushed during smpcfd_dying_cpu() which is invoked on the target CPU with disabled interrupts. The callbacks enqueued on lazy_list on PREEMPT_RT are not flushed because they require thread context. The callbacks remain and get processed once the CPU gets back online. per-CPU irq_work items continue to work for enqueues on their local-CPU, only the work on the "offline" CPU get stuck. "Single" irq_work are worse because they remain "claimed" and can not be used again. One "symptom" is the late printk() in CPU down path which enqueues the irq_work for printing but it never gets scheduled. Further prints don't print to the console because the irq_work is "pending" of the offline CPU. Another problem is the canceling/ flushing of the irq_work which is stuck. Since the irq_work will not continue, the task waiting for its completion will block waiting. While looking through the users of irq_work, it does not matter if the callback is enqueued on another CPU. This makes it possible to invoke the callbacks from a remote CPU by smpcfd_dead_cpu(). This is invoked from the control CPU with enabled interrupts shortly after the hotplugged CPU is shutdown so the llist can be accessed safely. Add irq_work_run_cpu() which invokes the irq_work callbacks of the specified dead CPU. Invoke it from smpcfd_dead_cpu(). Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911143815.997254-3-bigeasy@linutronix.de --- include/linux/irq_work.h | 2 ++ kernel/irq_work.c | 11 +++++++++++ kernel/smp.c | 1 + 3 files changed, 14 insertions(+) diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index c5afd053ae32..7184e1273965 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -59,6 +59,7 @@ void irq_work_sync(struct irq_work *work); #include void irq_work_run(void); +void irq_work_run_cpu(unsigned int cpu); bool irq_work_needs_cpu(void); void irq_work_single(void *arg); @@ -67,6 +68,7 @@ void arch_irq_work_raise(void); #else static inline bool irq_work_needs_cpu(void) { return false; } static inline void irq_work_run(void) { } +static inline void irq_work_run_cpu(unsigned int cpu) { } static inline void irq_work_single(void *arg) { } #endif diff --git a/kernel/irq_work.c b/kernel/irq_work.c index 9f19c05c9962..73eabcbdcd50 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -263,6 +263,17 @@ void irq_work_run(void) } EXPORT_SYMBOL_GPL(irq_work_run); +void irq_work_run_cpu(unsigned int cpu) +{ + if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, cpu_dying_mask))) + return; + + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return; + + irq_work_run_list(per_cpu_ptr(&lazy_list, cpu)); +} + void irq_work_tick(void) { struct llist_head *raised = this_cpu_ptr(&raised_list); diff --git a/kernel/smp.c b/kernel/smp.c index b696bcc60c08..8f530a094c30 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -87,6 +87,7 @@ int smpcfd_dead_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); + irq_work_run_cpu(cpu); return 0; } From 4a3b51aab6e25244d97936aa65e6d5425adf98e1 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 16:38:14 +0200 Subject: [PATCH 53/64] smpboot: Don't park the thread if work is pending Once a smpboot thread gets work assigned it receives a wake up. If the thread had no chance to run and receives a parking requesst during a CPU-hotplug event then it is parked without running the callback first. As a result the enqueued work remains stuck until the CPU gets back online. There is nothing wrong with invoking the thread function first and parking in the following iteration. This would ensure that the callbacks are processed before the shutdown. Honor the park request once thread function does not need to run. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911143815.997254-4-bigeasy@linutronix.de --- kernel/smpboot.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/smpboot.c b/kernel/smpboot.c index 4503b60ce9bd..3f60e8c6dd30 100644 --- a/kernel/smpboot.c +++ b/kernel/smpboot.c @@ -103,6 +103,7 @@ static int smpboot_thread_fn(void *data) { struct smpboot_thread_data *td = data; struct smp_hotplug_thread *ht = td->ht; + bool should_run; while (1) { set_current_state(TASK_INTERRUPTIBLE); @@ -117,7 +118,8 @@ static int smpboot_thread_fn(void *data) return 0; } - if (kthread_should_park()) { + should_run = td->status == HP_THREAD_ACTIVE && ht->thread_should_run(td->cpu); + if (kthread_should_park() && !should_run) { __set_current_state(TASK_RUNNING); preempt_enable(); if (ht->park && td->status == HP_THREAD_ACTIVE) { @@ -151,7 +153,7 @@ static int smpboot_thread_fn(void *data) continue; } - if (!ht->thread_should_run(td->cpu)) { + if (!should_run) { preempt_enable_no_resched(); schedule(); } else { From 9fc91ae6852aa6e63c08b6093634cfcc85021b4e Mon Sep 17 00:00:00 2001 From: Yiwei Lin Date: Mon, 5 Oct 2026 17:32:33 +0800 Subject: [PATCH 54/64] rbtree_test: Use rb_add() and rb_add_cached() for the basic tests insert() and insert_cached() open-code exactly what the generic rb_add() and rb_add_cached() helpers do. Use the helpers instead. Besides removing duplicated code, this makes rbtree_test cover the helpers themselves, which is what most in-tree rbtree users call nowadays. Signed-off-by: Yiwei Lin Assisted-by: LLM Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20261005093236.62702-2-s921975628@gmail.com --- lib/rbtree_test.c | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c index 768c5e6453f3..c1386d328cfb 100644 --- a/lib/rbtree_test.c +++ b/lib/rbtree_test.c @@ -30,41 +30,20 @@ static struct test_node *nodes = NULL; static struct rnd_state rnd; +static inline bool less(struct rb_node *a, const struct rb_node *b) +{ + return rb_entry(a, struct test_node, rb)->key < + rb_entry(b, struct test_node, rb)->key; +} + static void insert(struct test_node *node, struct rb_root_cached *root) { - struct rb_node **new = &root->rb_root.rb_node, *parent = NULL; - u32 key = node->key; - - while (*new) { - parent = *new; - if (key < rb_entry(parent, struct test_node, rb)->key) - new = &parent->rb_left; - else - new = &parent->rb_right; - } - - rb_link_node(&node->rb, parent, new); - rb_insert_color(&node->rb, &root->rb_root); + rb_add(&node->rb, &root->rb_root, less); } static void insert_cached(struct test_node *node, struct rb_root_cached *root) { - struct rb_node **new = &root->rb_root.rb_node, *parent = NULL; - u32 key = node->key; - bool leftmost = true; - - while (*new) { - parent = *new; - if (key < rb_entry(parent, struct test_node, rb)->key) - new = &parent->rb_left; - else { - new = &parent->rb_right; - leftmost = false; - } - } - - rb_link_node(&node->rb, parent, new); - rb_insert_color_cached(&node->rb, root, leftmost); + rb_add_cached(&node->rb, root, less); } static inline void erase(struct test_node *node, struct rb_root_cached *root) From c2c620f4345c50006af90c37fd104fed8e9dea14 Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Mon, 5 Oct 2026 17:32:34 +0800 Subject: [PATCH 55/64] rbtree: Declare augmented callbacks per field with RB_AUG() RB_DECLARE_CALLBACKS_MULTI() asks its user for a function that copies the augmented fields and one that recomputes them from the children, with the early-exit protocol of ->propagate() hand-coded in the latter. sched/eevdf, its only user, needs five helpers for three fields. Describe each augmented field instead: RB_AUG(val, aug, fold) names the per-node value, the member holding the subtree aggregate and how two aggregates combine (min, max, a sum, ...). RB_AUG_FUNC() takes a function for the per-node value. The template then generates, per field, a recompute that works on a local and stores once, and a copy, and combines them into the callbacks; the early exit is the AND of the per-field results. RB_DECLARE_CALLBACKS() is the only template left. RB_DECLARE_CALLBACKS_MAX() becomes RB_AUG_FUNC(RBVALUE, RBAUGMENTED, max) in a wrapper, with its RBCOMPUTE argument renamed to RBVALUE, since it returns the per-node scalar and is a different thing from the RBCOMPUTE of the generic template it used to build on. RB_DECLARE_CALLBACKS_MULTI() goes away. sched/eevdf shrinks to a wrapping-safe min() for min_vruntime and three RB_AUG() lines. No functional change intended. [yiwei: take a fold(a, b) instead of a "replace?" compare so that sums and counts can be expressed too, which also lets min()/max() replace RB_MIN()/RB_MAX(); wrapped the lines over 100 columns] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Yiwei Lin Assisted-by: LLM Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20260928213736.GA2947991@noisy.programming.kicks-ass.net Link: https://patch.msgid.link/20261005093236.62702-3-s921975628@gmail.com --- include/linux/rbtree_augmented.h | 174 +++++++++++++++++++++---------- kernel/sched/fair.c | 70 ++----------- 2 files changed, 126 insertions(+), 118 deletions(-) diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index d2fa1c41bfd2..5a336ac203c1 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -15,6 +15,8 @@ #include #include #include +#include +#include /* * Please note - only struct rb_augment_callbacks and the prototypes for @@ -86,6 +88,95 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, return leftmost ? node : NULL; } +#define RB_FOR_EACH_1(what, RBNAME, RBSTRUCT, RBFIELD, x) \ + what(1, RBNAME, RBSTRUCT, RBFIELD, x) + +#define RB_FOR_EACH_2(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(2, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_1(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_3(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(3, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_2(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_4(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(4, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_3(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_5(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(5, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_4(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_6(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(6, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_5(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_7(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(7, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_6(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_8(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(8, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_7(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH(action, RBNAME, RBSTRUCT, RBFIELD, ...) \ + CONCATENATE(RB_FOR_EACH_, COUNT_ARGS(__VA_ARGS__)) \ + (action, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +/* + * One augmented field: @val is the node's own contribution (a member for + * RB_AUG(), a function of the node for RB_AUG_FUNC()), @aug the member + * holding the aggregate over the subtree, and @fold(a, b) combines two + * aggregates: min, max, a sum, ... It must be commutative and associative. + */ +#define RB_AUG_FUNC(val, aug, fold) (val(s), aug, fold) +#define RB_AUG(val, aug, fold) (s->val, aug, fold) +#define RB_UNPACK(...) __VA_ARGS__ + +#define __RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ +static inline void \ +RBNAME ## _copy_ ## n(RBSTRUCT *old, RBSTRUCT *new) \ +{ \ + new->aug = old->aug; \ +} \ +static inline bool \ +RBNAME ## _compute_ ## n(RBSTRUCT *s, bool exit) \ +{ \ + TYPEOF_UNQUAL(s->aug) _old_aug = s->aug; \ + TYPEOF_UNQUAL(s->aug) _val = val; \ + struct rb_node *_node = &s->RBFIELD; \ + if (_node->rb_right) { \ + RBSTRUCT *_c = container_of(_node->rb_right, typeof(*s), RBFIELD); \ + _val = fold(_val, _c->aug); \ + } \ + if (_node->rb_left) { \ + RBSTRUCT *_c = container_of(_node->rb_left, typeof(*s), RBFIELD); \ + _val = fold(_val, _c->aug); \ + } \ + if (exit && _old_aug == _val) \ + return true; \ + s->aug = _val; \ + return false; \ +} +#define _RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + +#define __RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + RBNAME ## _copy_ ## n(old, new); +#define _RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + +#define __RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + ret &= RBNAME ## _compute_ ## n(node, exit); +#define _RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + /* * Template for declaring augmented rbtree callbacks (generic multi fields) * @@ -93,18 +184,29 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBCOPY: name of function that copies the RBAUGMENTED datas - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas + * RBAUG...: list of RB_AUG() describing the augmented data */ - -#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUG...) \ +RB_FOR_EACH(RB_INST, RBNAME, RBSTRUCT, RBFIELD, RBAUG) \ +static inline void \ +RBNAME ## __copy(RBSTRUCT *old, RBSTRUCT *new) \ +{ \ + RB_FOR_EACH(RB_COPY, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ +} \ +static inline bool \ +RBNAME ## __compute(RBSTRUCT *node, bool exit) \ +{ \ + bool ret = true; \ + RB_FOR_EACH(RB_COMPUTE, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ + return ret; \ +} \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ while (rb != stop) { \ RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \ - if (RBCOMPUTE(node, true)) \ + if (RBNAME ## __compute(node, true)) \ break; \ rb = rb_parent(&node->RBFIELD); \ } \ @@ -114,15 +216,15 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - RBCOPY(new, old); \ + RBNAME ## __copy(old, new); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - RBCOPY(new, old); \ - RBCOMPUTE(old, false); \ + RBNAME ## __copy(old, new); \ + RBNAME ## __compute(old, false); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .propagate = RBNAME ## _propagate, \ @@ -130,27 +232,6 @@ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .rotate = RBNAME ## _rotate \ }; -/* - * Template for declaring augmented rbtree callbacks (generic single field) - * - * RBSTATIC: 'static' or empty - * RBNAME: name of the rb_augment_callbacks structure - * RBSTRUCT: struct type of the tree nodes - * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data - */ - -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ -static inline void \ -RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ -{ \ - new->RBAUGMENTED = old->RBAUGMENTED; \ -} \ -RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) - /* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. @@ -159,34 +240,15 @@ RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBTYPE: type of the RBAUGMENTED field - * RBAUGMENTED: name of RBTYPE field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that returns the per-node RBTYPE scalar + * RBTYPE: type of the RBAUGMENTED field -- unused, assumed typeof(RBAUGMENTED) + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBVALUE: name of function that returns the per-node RBTYPE scalar */ -#define RB_DECLARE_CALLBACKS_MAX(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ - RBTYPE, RBAUGMENTED, RBCOMPUTE) \ -static inline bool RBNAME ## _compute_max(RBSTRUCT *node, bool exit) \ -{ \ - RBSTRUCT *child; \ - RBTYPE max = RBCOMPUTE(node); \ - if (node->RBFIELD.rb_left) { \ - child = rb_entry(node->RBFIELD.rb_left, RBSTRUCT, RBFIELD); \ - if (child->RBAUGMENTED > max) \ - max = child->RBAUGMENTED; \ - } \ - if (node->RBFIELD.rb_right) { \ - child = rb_entry(node->RBFIELD.rb_right, RBSTRUCT, RBFIELD); \ - if (child->RBAUGMENTED > max) \ - max = child->RBAUGMENTED; \ - } \ - if (exit && node->RBAUGMENTED == max) \ - return true; \ - node->RBAUGMENTED = max; \ - return false; \ -} \ -RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBNAME ## _compute_max) +#define RB_DECLARE_CALLBACKS_MAX(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ + RBTYPE, RBAUGMENTED, RBVALUE) \ +RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ + RB_AUG_FUNC(RBVALUE, RBAUGMENTED, max)) #define RB_RED 0 diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5c98d8dfce5d..a58d513b3d63 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -999,71 +999,17 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) return entity_before(__node_2_se(a), __node_2_se(b)); } -static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) +/* min() for wrapping vruntimes */ +static inline u64 __min_vruntime(u64 a, u64 b) { - if (node) { - struct sched_entity *rse = __node_2_se(node); - - if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) - se->min_vruntime = rse->min_vruntime; - } + return vruntime_cmp(a, "<", b) ? a : b; } -static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) -{ - if (node) { - struct sched_entity *rse = __node_2_se(node); - if (rse->min_slice < se->min_slice) - se->min_slice = rse->min_slice; - } -} - -static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) -{ - if (node) { - struct sched_entity *rse = __node_2_se(node); - if (rse->max_slice > se->max_slice) - se->max_slice = rse->max_slice; - } -} - -static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) -{ - new->min_vruntime = old->min_vruntime; - new->min_slice = old->min_slice; - new->max_slice = old->max_slice; -} - -/* - * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) - */ -static inline bool min_vruntime_update(struct sched_entity *se, bool exit) -{ - u64 old_min_vruntime = se->min_vruntime; - u64 old_min_slice = se->min_slice; - u64 old_max_slice = se->max_slice; - struct rb_node *node = &se->run_node; - - se->min_vruntime = se->vruntime; - __min_vruntime_update(se, node->rb_right); - __min_vruntime_update(se, node->rb_left); - - se->min_slice = se->slice; - __min_slice_update(se, node->rb_right); - __min_slice_update(se, node->rb_left); - - se->max_slice = se->slice; - __max_slice_update(se, node->rb_right); - __max_slice_update(se, node->rb_left); - - return se->min_vruntime == old_min_vruntime && - se->min_slice == old_min_slice && - se->max_slice == old_max_slice; -} - - -RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime_copy, min_vruntime_update); +RB_DECLARE_CALLBACKS(static, min_vruntime_cb, + struct sched_entity, run_node, + RB_AUG(vruntime, min_vruntime, __min_vruntime), + RB_AUG(slice, min_slice, min), + RB_AUG(slice, max_slice, max)); /* * Enqueue an entity into the rb-tree: From cd047867ee5fc7c47cf9461c8a59c7117de9f680 Mon Sep 17 00:00:00 2001 From: Yiwei Lin Date: Mon, 5 Oct 2026 17:32:35 +0800 Subject: [PATCH 56/64] rbtree: Update augmented data on the way down in rb_add_augmented_cached() rb_add_augmented_cached() links the new node and then calls ->propagate() from the parent up to the root to bring the augmented data of the ancestors up to date. It marks that as "suboptimal": every augmented user that open-codes its insertion (interval trees, vmalloc, drm_mm, gpu buddy, ...) instead folds the new node's value into each ancestor while walking down, which touches no sibling and needs no second pass. ->propagate() has to read both children of every ancestor until it finds one whose value did not change. Add a ->merge() callback to struct rb_augment_callbacks that folds the augmented data of a node being inserted into one of its future ancestors, and use it on the descent. RB_DECLARE_CALLBACKS() derives it from the RB_AUG() list, so every user of the template gets it for free, and sched/eevdf, the only caller of the helper, needs no change. The new node's augmented data must describe the node alone before the call. That was already required, since ->propagate() read it, and sched/eevdf sets it up that way. Measured with rbtree_test's cached augmented test switched to rb_add_augmented_cached() (next patch), nnodes cached insert+delete, median of 5 runs, for reference also the open-coded descent used by augmented test 1. Raspberry Pi 4 Model B (Cortex-A72, 1.8 GHz), arch timer ticks (54 MHz): nnodes open-coded before after 100 459 500 442 (-11.6%) 10000 182046 186144 180208 (-3.2%) 100000 5533679 5711647 5579444 (-2.3%) x86-64 KVM guest (Intel i7-13800H), cycles: nnodes open-coded before after 100 7657 7622 7059 (-7.4%) 10000 4738075 5075847 4627462 (-8.8%) 100000 75618429 79840335 71324700 (-10.7%) The other rbtree_test numbers are unchanged and the augmented invariant checks still pass on both. Signed-off-by: Yiwei Lin Assisted-by: LLM Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20261005093236.62702-4-s921975628@gmail.com --- Documentation/core-api/rbtree.rst | 27 ++++++++++++++++++++++++--- include/linux/rbtree_augmented.h | 27 +++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/Documentation/core-api/rbtree.rst b/Documentation/core-api/rbtree.rst index cce80e19087b..210045f128d6 100644 --- a/Documentation/core-api/rbtree.rst +++ b/Documentation/core-api/rbtree.rst @@ -257,8 +257,14 @@ When erasing a node, the user must call rb_erase_augmented() instead of rb_erase(). rb_erase_augmented() calls back into user provided functions to update the augmented information on affected subtrees. -In both cases, the callbacks are provided through struct rb_augment_callbacks. -3 callbacks must be defined: +Alternatively, rb_add_augmented_cached() performs the whole insertion into +a cached tree: it walks down to the insertion point, merges the new node's +augmented information into every node on that path, links the node and +rebalances. The new node's augmented information must already describe +the node alone when it is called. + +In all cases, the callbacks are provided through struct rb_augment_callbacks. +3 callbacks must be defined, plus a fourth one for rb_add_augmented_cached(): - A propagation callback, which updates the augmented value for a given node and its ancestors, up to a given stop point (or NULL to update @@ -271,6 +277,10 @@ In both cases, the callbacks are provided through struct rb_augment_callbacks. subtree to a newly assigned subtree root AND recomputes the augmented information for the former subtree root. +- A merge callback, which folds the augmented value of a node being inserted + into the augmented value of one of its future ancestors. It is only used + by rb_add_augmented_cached(). + The compiled code for rb_erase_augmented() may inline the propagation and copy callbacks, which results in a large function, so each augmented rbtree user should have a single rb_erase_augmented() call site in order to limit @@ -395,8 +405,19 @@ Insertion/removal are defined using the following augmented callbacks:: old->__subtree_last = compute_subtree_last(old); } + static void augment_merge(struct rb_node *rb, struct rb_node *rb_new) + { + struct interval_tree_node *node = + rb_entry(rb, struct interval_tree_node, rb); + struct interval_tree_node *new = + rb_entry(rb_new, struct interval_tree_node, rb); + + if (node->__subtree_last < new->__subtree_last) + node->__subtree_last = new->__subtree_last; + } + static const struct rb_augment_callbacks augment_callbacks = { - augment_propagate, augment_copy, augment_rotate + augment_propagate, augment_copy, augment_rotate, augment_merge }; void interval_tree_insert(struct interval_tree_node *node, diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 5a336ac203c1..b9fc1be86ae3 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -30,6 +30,7 @@ struct rb_augment_callbacks { void (*propagate)(struct rb_node *node, struct rb_node *stop); void (*copy)(struct rb_node *old, struct rb_node *new); void (*rotate)(struct rb_node *old, struct rb_node *new); + void (*merge)(struct rb_node *node, struct rb_node *new); }; extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, @@ -62,6 +63,12 @@ rb_insert_augmented_cached(struct rb_node *node, rb_insert_augmented(node, &root->rb_root, augment); } +/* + * Insert @node into the leftmost cached augmented tree @tree. + * + * The augmented data of @node must already describe @node alone; it is + * merged into every ancestor on the way down through augment->merge(). + */ static __always_inline struct rb_node * rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, bool (*less)(struct rb_node *, const struct rb_node *), @@ -73,6 +80,7 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, while (*link) { parent = *link; + augment->merge(parent, node); if (less(node, parent)) { link = &parent->rb_left; } else { @@ -82,7 +90,6 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } rb_link_node(node, parent, link); - augment->propagate(parent, NULL); /* suboptimal */ rb_insert_augmented_cached(node, tree, leftmost, augment); return leftmost ? node : NULL; @@ -177,6 +184,14 @@ RBNAME ## _compute_ ## n(RBSTRUCT *s, bool exit) \ #define RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, x) \ _RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) +/* fold @new, about to become a descendant of @node, into @node */ +#define __RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + node->aug = fold(node->aug, new->aug); +#define _RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + /* * Template for declaring augmented rbtree callbacks (generic multi fields) * @@ -226,10 +241,18 @@ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ RBNAME ## __copy(old, new); \ RBNAME ## __compute(old, false); \ } \ +static inline void \ +RBNAME ## _merge(struct rb_node *rb, struct rb_node *rb_new) \ +{ \ + RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \ + RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ + RB_FOR_EACH(RB_MERGE, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ +} \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .propagate = RBNAME ## _propagate, \ .copy = RBNAME ## _copy, \ - .rotate = RBNAME ## _rotate \ + .rotate = RBNAME ## _rotate, \ + .merge = RBNAME ## _merge \ }; /* From cc459ecd9388d96dc20404b157f5a4eedb66c355 Mon Sep 17 00:00:00 2001 From: Yiwei Lin Date: Mon, 5 Oct 2026 17:32:36 +0800 Subject: [PATCH 57/64] rbtree_test: Use rb_add_augmented_cached() for the cached augmented test insert_augmented_cached() open-codes the descent that the generic rb_add_augmented_cached() helper provides. Use the helper instead, so that rbtree_test covers it: it is what sched/eevdf relies on and nothing in lib/ exercises it today. Signed-off-by: Yiwei Lin Assisted-by: LLM Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20261005093236.62702-5-s921975628@gmail.com --- lib/rbtree_test.c | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c index c1386d328cfb..00f62e4f9d10 100644 --- a/lib/rbtree_test.c +++ b/lib/rbtree_test.c @@ -89,29 +89,8 @@ static void insert_augmented(struct test_node *node, static void insert_augmented_cached(struct test_node *node, struct rb_root_cached *root) { - struct rb_node **new = &root->rb_root.rb_node, *rb_parent = NULL; - u32 key = node->key; - u32 val = node->val; - struct test_node *parent; - bool leftmost = true; - - while (*new) { - rb_parent = *new; - parent = rb_entry(rb_parent, struct test_node, rb); - if (parent->augmented < val) - parent->augmented = val; - if (key < parent->key) - new = &parent->rb.rb_left; - else { - new = &parent->rb.rb_right; - leftmost = false; - } - } - - node->augmented = val; - rb_link_node(&node->rb, rb_parent, new); - rb_insert_augmented_cached(&node->rb, root, - leftmost, &augment_callbacks); + node->augmented = node->val; + rb_add_augmented_cached(&node->rb, root, less, &augment_callbacks); } From 159b172906110f736fdbd7af8a21d0e0b469f671 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Sun, 11 Oct 2026 10:50:19 +0900 Subject: [PATCH 58/64] sched/wait: Avoid modifying entry on failed priority registration add_wait_queue_priority_exclusive() sets WQ_FLAG_EXCLUSIVE and WQ_FLAG_PRIORITY before checking whether another priority waiter is already present. Thus, when registration fails with -EBUSY, the entry is left unqueued but with modified flags. There is no need to modify the entry before the failure check. Move the flag update after the check so that a rejected registration does not unnecessarily alter the entry. Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sean Christopherson Link: https://patch.msgid.link/20261011015019.4056529-1-sh_def@163.com --- kernel/sched/wait.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 477e4bf9c01e..f8d6d8b9d03e 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -54,14 +54,13 @@ int add_wait_queue_priority_exclusive(struct wait_queue_head *wq_head, { struct list_head *head = &wq_head->head; - wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY; - guard(spinlock_irqsave)(&wq_head->lock); if (!list_empty(head) && (list_first_entry(head, typeof(*wq_entry), entry)->flags & WQ_FLAG_PRIORITY)) return -EBUSY; + wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY; list_add(&wq_entry->entry, head); return 0; } From b0f1d7a34d3099973f1ba45d1ce6f36578e510a2 Mon Sep 17 00:00:00 2001 From: Elif Topuz Date: Mon, 12 Oct 2026 11:50:01 +0100 Subject: [PATCH 59/64] sched/eevdf: Remove redundant RUN_TO_PARITY check update_protect_slice() is only called from wakeup_preempt_fair() when RUN_TO_PARITY is enabled. Remove RUN_TO_PARITY check inside update_protect_slice() and use cfs_rq_min_slice() directly. Signed-off-by: Elif Topuz Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20261012105001.131311-1-elif.topuz@arm.com --- kernel/sched/fair.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a58d513b3d63..7bf31931fe09 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1089,13 +1089,8 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); - u64 slice = normalized_sysctl_sched_base_slice; - u64 vprot; - - if (sched_feat(RUN_TO_PARITY)) - slice = cfs_rq_min_slice(cfs_rq); - - vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); + u64 slice = cfs_rq_min_slice(cfs_rq); + u64 vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); if (sched_feat(PREEMPT_SHORT) && slice != se->slice) vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); From 6f88f6cbb0472a8481ba560578a1d7c8d802eba7 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Sat, 10 Oct 2026 12:15:00 -0700 Subject: [PATCH 60/64] sched/psi: Remove the RCU wait from trigger destruction In the Meta fleet, we found PSI trigger destruction waiting for an RCU grace period while holding kernfs node_mutex, blocking other users [1]. File close and node drain hold this mutex while calling the release callback. Cgroup removal and disabling cgroup.pressure also hold cgroup_mutex during the drain. The wait was added by commit 0e94682b73bf ("psi: introduce psi monitor") to protect worker and trigger lookups. Commit 461daba06bdc ("psi: eliminate kthread_worker from psi trigger scheduling mechanism") replaced the worker queue with a group timer. The scheduler now only checks whether rtpoll_task is NULL; it uses neither the task nor the trigger. Commit a06247c6804f ("psi: Fix uaf issue when psi trigger is destroyed while being polled") removed trigger replacement and polling's RCU lookup. Trigger lists are protected by mutexes. Proc poll/select entries are removed before their file references are dropped. eventpoll_release() removes epoll entries before the file release callback. Commit aff037078eca ("sched/psi: use kernfs polling functions for PSI trigger polling") moved cgroup polling to a kernfs waitqueue whose lifetime follows the file. A late timer firing only wakes the group waitqueue. The system group is static, and commit 5457025fa8ca ("sched/psi: Shut down rtpoll_timer in psi_cgroup_free()") shuts down the cgroup timer before freeing the group. Cgroup reclamation already waits for scheduler readers. Remove synchronize_rcu() from psi_trigger_destroy() and use rcu_access_pointer() for the worker NULL check. Keep kthread_stop() outside the trigger mutex. Tested trigger churn, notifications and cgroup removal in an 8-CPU VM with KASAN, lockdep, and full and lazy preemption. No warnings were found. Signed-off-by: Shakeel Butt Signed-off-by: Peter Zijlstra (Intel) Acked-by: Tejun Heo Link: https://github.com/bpftrace/user-tools/tree/master/runnablelockmonitor [1] Link: https://patch.msgid.link/20261010191500.3895736-1-shakeel.butt@linux.dev --- kernel/sched/psi.c | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 4e152410653d..7c5381423334 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -626,8 +626,6 @@ static void init_rtpoll_triggers(struct psi_group *group, u64 now) static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay, bool force) { - struct task_struct *task; - /* * atomic_xchg should be called even when !force to provide a * full memory barrier (see the comment inside psi_rtpoll_work). @@ -635,19 +633,16 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela if (atomic_xchg(&group->rtpoll_scheduled, 1) && !force) return; - rcu_read_lock(); - - task = rcu_dereference(group->rtpoll_task); /* - * kworker might be NULL in case psi_trigger_destroy races with - * psi_task_change (hotpath) which can't use locks + * Only test whether a worker is installed; do not dereference the task. + * A racing trigger destruction may leave the timer armed. + * psi_cgroup_free() shuts it down before freeing the group. + * The system PSI group is static. */ - if (likely(task)) + if (likely(rcu_access_pointer(group->rtpoll_task))) mod_timer(&group->rtpoll_timer, jiffies + delay); else atomic_set(&group->rtpoll_scheduled, 0); - - rcu_read_unlock(); } static void psi_rtpoll_work(struct psi_group *group) @@ -1488,22 +1483,12 @@ void psi_trigger_destroy(struct psi_trigger *t) mutex_unlock(&group->rtpoll_trigger_lock); } - /* - * Wait for psi_schedule_rtpoll_work RCU to complete its read-side - * critical section before destroying the trigger and optionally the - * rtpoll_task. - */ - synchronize_rcu(); /* * Stop kthread 'psimon' after releasing rtpoll_trigger_lock to prevent * a deadlock while waiting for psi_rtpoll_work to acquire * rtpoll_trigger_lock */ if (task_to_destroy) { - /* - * After the RCU grace period has expired, the worker - * can no longer be found through group->rtpoll_task. - */ kthread_stop(task_to_destroy); atomic_set(&group->rtpoll_scheduled, 0); } From 96a494c9cfd8f4bb7b1a7a70275426db71cf98aa Mon Sep 17 00:00:00 2001 From: Stian Halseth Date: Wed, 7 Oct 2026 16:29:50 +0200 Subject: [PATCH 61/64] sched/cputime: Don't account idle time twice after dyntick-idle On idle exit the dyntick-idle accounting accounts the time up to now, then the tick is restarted on its old period. The first tick accounts a whole TICK_NSEC to whatever runs, although the part of that period before the idle exit has just been accounted as idle time, or with IRQ_TIME_ACCOUNTING as IRQ time. That is up to a full tick per idle exit, and /proc/stat reports more idle time than wall time. Record how much of the current tick period dyntick-idle has accounted, and leave it out of the tick that ends the period. Fixes: cf6444c3e1bb7 ("tick/sched: Unify idle cputime accounting") Signed-off-by: Stian Halseth Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Frederic Weisbecker Link: https://lore.kernel.org/all/20261004142724.3896396-1-stian@itx.no/ Link: https://patch.msgid.link/20261007142950.547604-1-stian@itx.no --- include/linux/kernel_stat.h | 7 +++++-- kernel/sched/cputime.c | 39 +++++++++++++++++++++++++++++++------ kernel/time/tick-sched.c | 22 ++++++++++++++++----- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index c1e85550bf12..9f0f91dc8d7d 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -40,6 +40,9 @@ struct kernel_cpustat { seqcount_t idle_sleeptime_seq; u64 idle_entrytime; u64 idle_stealtime[2]; + u64 idle_dyntick_entry; + u64 idle_tick_period; + u64 idle_tick_overlap; #endif u64 cpustat[NR_STATS]; }; @@ -111,7 +114,7 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) #ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE static inline void kcpustat_dyntick_start(u64 now) { } -static inline void kcpustat_dyntick_stop(u64 now) { } +static inline void kcpustat_dyntick_stop(u64 now, u64 tick_start) { } static inline void kcpustat_irq_enter(u64 now) { } static inline void kcpustat_irq_exit(u64 now) { } static inline bool kcpustat_idle_dyntick(void) { return false; } @@ -132,7 +135,7 @@ static inline u64 kcpustat_field_iowait(int cpu) #else /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ extern void kcpustat_dyntick_start(u64 now); -extern void kcpustat_dyntick_stop(u64 now); +extern void kcpustat_dyntick_stop(u64 now, u64 tick_start); extern void kcpustat_irq_enter(u64 now); extern void kcpustat_irq_exit(u64 now); extern u64 kcpustat_field_idle(int cpu); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index f16970ca81d0..19aa804c2589 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -381,9 +381,9 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) * softirq as those do not count in task exec_runtime any more. */ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, - int ticks) + u64 cputime) { - u64 other, cputime = TICK_NSEC * ticks; + u64 other; /* * When returning from idle, many ticks can get accounted at @@ -418,7 +418,7 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick, - int nr_ticks) { } + u64 cputime) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ #if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) @@ -468,18 +468,38 @@ static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) write_seqcount_end(&kc->idle_sleeptime_seq); } -void kcpustat_dyntick_stop(u64 now) +void kcpustat_dyntick_stop(u64 now, u64 tick_start) { struct kernel_cpustat *kc = kcpustat_this_cpu; if (!vtime_generic_enabled_this_cpu()) { WARN_ON_ONCE(!kc->idle_dyntick); + if (kc->idle_tick_period != tick_start) { + kc->idle_tick_period = tick_start; + kc->idle_tick_overlap = 0; + } + tick_start = max(tick_start, kc->idle_dyntick_entry); + if (!WARN_ON_ONCE(now < tick_start)) + kc->idle_tick_overlap += now - tick_start; kcpustat_idle_stop(kc, now); kc->idle_dyntick = false; vtime_dyntick_stop(); } } +/* + * The first tick after dyntick-idle covers a period that the dyntick-idle + * accounting may already have accounted in part. + */ +static inline u64 kcpustat_tick_overlap(void) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + u64 overlap = kc->idle_tick_overlap; + + kc->idle_tick_overlap = 0; + return overlap; +} + void kcpustat_dyntick_start(u64 now) { struct kernel_cpustat *kc = kcpustat_this_cpu; @@ -487,6 +507,7 @@ void kcpustat_dyntick_start(u64 now) if (!vtime_generic_enabled_this_cpu()) { vtime_dyntick_start(); kc->idle_dyntick = true; + kc->idle_dyntick_entry = now; kcpustat_idle_start(kc, now); } } @@ -555,6 +576,11 @@ u64 kcpustat_field_iowait(int cpu) } EXPORT_SYMBOL_GPL(kcpustat_field_iowait); #else +static inline u64 kcpustat_tick_overlap(void) +{ + return 0; +} + static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, bool compute_delta, ktime_t now) { @@ -695,12 +721,13 @@ void account_process_tick(struct task_struct *p, int user_tick) if (kcpustat_idle_dyntick()) return; + cputime = TICK_NSEC - kcpustat_tick_overlap(); + if (irqtime_enabled()) { - irqtime_account_process_tick(p, user_tick, 1); + irqtime_account_process_tick(p, user_tick, cputime); return; } - cputime = TICK_NSEC; steal = steal_account_process_time(ULONG_MAX); if (steal >= cputime) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 6c3fea386713..4e293f81e4dc 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -763,12 +763,18 @@ static ktime_t tick_forward_now(ktime_t expires, ktime_t now) return expires + TICK_NSEC; } +static void tick_nohz_forward_last_tick(struct tick_sched *ts, ktime_t now) +{ + if (now >= ts->last_tick) + ts->last_tick = tick_forward_now(ts->last_tick, now); +} + static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) { - ktime_t expires = ts->last_tick; + ktime_t expires; - if (now >= expires) - expires = tick_forward_now(expires, now); + tick_nohz_forward_last_tick(ts, now); + expires = ts->last_tick; if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) { hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED_HARD); @@ -1329,6 +1335,12 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu) return ts->idle_calls; } +static void tick_nohz_dyntick_stop(struct tick_sched *ts, ktime_t now) +{ + tick_nohz_forward_last_tick(ts, now); + kcpustat_dyntick_stop(now, ts->last_tick - TICK_NSEC); +} + void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); @@ -1341,7 +1353,7 @@ void tick_nohz_idle_restart_tick(void) * no tiny amount of idle time is accounted twice. */ ts->idle_entrytime = ktime_get(); - kcpustat_dyntick_stop(ts->idle_entrytime); + tick_nohz_dyntick_stop(ts, ts->idle_entrytime); tick_nohz_restart_sched_tick(ts, ts->idle_entrytime); } } @@ -1385,7 +1397,7 @@ void tick_nohz_idle_exit(void) if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { now = ktime_get(); - kcpustat_dyntick_stop(now); + tick_nohz_dyntick_stop(ts, now); tick_nohz_idle_update_tick(ts, now); } From fcceb7da94659695d6cba97190e18e87d9cce82c Mon Sep 17 00:00:00 2001 From: Jemmy Wong Date: Tue, 13 Oct 2026 16:43:38 +0800 Subject: [PATCH 62/64] sched/cache: Avoid publishing partial LLC utilization update_sg_lb_stats() sums utilization over the CPUs in both sched_group_span(group) and env->cpus, but takes capacity from group->sgc->capacity for the whole group. record_sg_llc_stats() publishes both. Rejecting an all-affinity-pinned source runqueue removes that CPU from env->cpus and retries group selection. The retry can publish the partial utilization with the full capacity. can_migrate_llc() then treats a busy preferred LLC as having spare capacity. More tasks can aggregate into it, migrations out can be blocked, and two busy LLCs may not fall back to generic balancing. This increases CPU contention and task latency. Found by inspection. Skip publication on affinity retries using LBF_ALL_PINNED, retaining the first-pass snapshots and avoiding repeated cpumask checks. On the first pass, publish either shared LLC statistic only when env->cpus covers the whole group. Load balancing still uses the affinity-filtered statistics. The same window exists during CPU hotplug. env->cpus drops a CPU at set_cpu_active(false), but the group span and sgc->capacity still include it until cpuset_cpu_inactive() rebuilds the domains. Keeping the previous snapshot avoids the mismatched publish. After the rebuild, the next periodic balance records both statistics again. Fixes: f025ef275388 ("sched/cache: Record per LLC utilization to guide cache aware scheduling decisions") Suggested-by: K Prateek Nayak Suggested-by: Tim Chen Signed-off-by: Jemmy Wong Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20261013084338.55100-1-jemmywong512@gmail.com --- kernel/sched/fair.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7bf31931fe09..a90e48d92058 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12083,6 +12083,14 @@ static void record_sg_llc_stats(struct lb_env *env, if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu))) return; + /* + * A redo repeats stats published on the first pass. Hotplug can + * still leave env->cpus partial, so util would not match capacity. + */ + if ((env->flags & LBF_ALL_PINNED) || + !cpumask_subset(sched_group_span(group), env->cpus)) + return; + /* * At this point we know this group spans a LLC domain. * Record the statistic of this group in its corresponding From 0a94d0ea8d8b464ff597ae2f534dd60b64b4da0f Mon Sep 17 00:00:00 2001 From: Kayra Cizmeci Date: Thu, 15 Oct 2026 11:29:18 +0300 Subject: [PATCH 63/64] Revert "sched/fair: Remove dead code on enqueue_task_fair()" This reverts commit 819224e506bc7c2d61ec6a58ec6e876b505abce9. The cfs_rq->curr == se case can actually happen with CFS_BANDWIDTH, so the removed code is needed. Document this. Fixes: 819224e506bc ("sched/fair: Remove dead code on enqueue_task_fair()") Reported-by: Christian Loehle Signed-off-by: Kayra Cizmeci Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/lkml/fd3f9d83-cb5b-496b-b7c1-b40f31cf795e@arm.com/ Link: https://patch.msgid.link/20261015082919.4344-1-kayracizmeci@gmail.com --- kernel/sched/fair.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a90e48d92058..0feec172fd37 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8156,6 +8156,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) struct sched_entity *se = &p->se; struct cfs_rq *cfs_rq = &rq->cfs; unsigned long weight; + bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -8184,14 +8185,25 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); + curr = (cfs_rq->curr == se); + /* + * This can actually happen with CFS_BANDWIDTH, a task can be throttled + * and dequeued, unthrottled from another CPU and then enqueued again + * before schedule(). + */ + if (curr) + place_entity(cfs_rq, se, flags); if (se->on_rq && se->sched_delayed) requeue_delayed_entity(cfs_rq, se); weight = enqueue_hierarchy(p, flags); - reweight_eevdf(cfs_rq, se, weight, false); - place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); - __enqueue_entity(cfs_rq, se); + + if (!curr) { + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); + } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) dl_server_start(&rq->fair_server); From b99b7f60c5b473d374c68b53753153b4aa9e0702 Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Fri, 16 Oct 2026 08:33:40 +0100 Subject: [PATCH 64/64] sched: Replace nr_iowait_cpu() with a bool accessor The menu governor was the last consumer of the per-CPU I/O-wait count that needed its magnitude, apart from nr_iowait() summing it globally. Commit 38f83090f515 ("cpuidle: menu: Remove iowait influence") removed that use. All remaining per-CPU consumers only test for zero/nonzero. Replace nr_iowait_cpu() with bool cpu_has_iowait() and update the generic and s390 idle accounting callers. Have nr_iowait() read the counters directly so that the global total and existing accounting are preserved. Replace the stale menu comment with a description of the predicate. Also convert the remaining open-coded idle accounting checks to use the predicate, as suggested by Prateek. The conversion was done with coccinelle: @ predicate depends on patch @ expression cpu; @@ - nr_iowait_cpu(cpu) > 0 + cpu_has_iowait(cpu) @ callers depends on patch @ expression cpu; @@ - nr_iowait_cpu(cpu) + cpu_has_iowait(cpu) Signed-off-by: Christian Loehle Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kayra Cizmeci Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Link: https://patch.msgid.link/20261016073341.575472-1-christian.loehle@arm.com --- arch/s390/kernel/idle.c | 4 ++-- include/linux/sched/stat.h | 2 +- kernel/sched/core.c | 13 +++++-------- kernel/sched/cputime.c | 20 +++++++++----------- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index dd12efb58201..ff9732ef494c 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -82,13 +82,13 @@ static u64 arch_cpu_idle_time(int cpu, enum cpu_usage_stat idx, bool compute_del u64 arch_kcpustat_field_idle(int cpu) { - return arch_cpu_idle_time(cpu, CPUTIME_IDLE, !nr_iowait_cpu(cpu)); + return arch_cpu_idle_time(cpu, CPUTIME_IDLE, !cpu_has_iowait(cpu)); } EXPORT_SYMBOL_GPL(arch_kcpustat_field_idle); u64 arch_kcpustat_field_iowait(int cpu) { - return arch_cpu_idle_time(cpu, CPUTIME_IOWAIT, nr_iowait_cpu(cpu)); + return arch_cpu_idle_time(cpu, CPUTIME_IOWAIT, cpu_has_iowait(cpu)); } EXPORT_SYMBOL_GPL(arch_kcpustat_field_iowait); diff --git a/include/linux/sched/stat.h b/include/linux/sched/stat.h index 0108a38bb64d..4810b14d0372 100644 --- a/include/linux/sched/stat.h +++ b/include/linux/sched/stat.h @@ -20,7 +20,7 @@ extern int nr_processes(void); extern unsigned int nr_running(void); extern bool single_task_running(void); extern unsigned int nr_iowait(void); -extern unsigned int nr_iowait_cpu(int cpu); +extern bool cpu_has_iowait(int cpu); static inline int sched_info_on(void) { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 047fbe8ef298..d7c3b43158de 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5638,15 +5638,12 @@ unsigned long long nr_context_switches(void) } /* - * Consumers of these two interfaces, like for example the cpuidle menu - * governor, are using nonsensical data. Preferring shallow idle state selection - * for a CPU that has IO-wait which might not even end up running the task when - * it does become runnable. + * Return whether any tasks are accounted as I/O waiters on this CPU. + * They may wake up on a different CPU. */ - -unsigned int nr_iowait_cpu(int cpu) +bool cpu_has_iowait(int cpu) { - return atomic_read(&cpu_rq(cpu)->nr_iowait); + return atomic_read(&cpu_rq(cpu)->nr_iowait) != 0; } /* @@ -5684,7 +5681,7 @@ unsigned int nr_iowait(void) unsigned int i, sum = 0; for_each_possible_cpu(i) - sum += nr_iowait_cpu(i); + sum += atomic_read(&cpu_rq(i)->nr_iowait); return sum; } diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 19aa804c2589..9e0fb8633d82 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -226,9 +226,8 @@ void account_steal_time(u64 cputime) void account_idle_time(u64 cputime) { u64 *cpustat = kcpustat_this_cpu->cpustat; - struct rq *rq = this_rq(); - if (atomic_read(&rq->nr_iowait) > 0) + if (cpu_has_iowait(smp_processor_id())) cpustat[CPUTIME_IOWAIT] += cputime; else cpustat[CPUTIME_IDLE] += cputime; @@ -431,7 +430,7 @@ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) if (!kc->idle_elapse) return; - iowait = nr_iowait_cpu(smp_processor_id()) > 0; + iowait = cpu_has_iowait(smp_processor_id()); delta = now - kc->idle_entrytime; steal = steal_account_process_time(delta); @@ -565,14 +564,14 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, u64 kcpustat_field_idle(int cpu) { return kcpustat_field_dyntick(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), ktime_get()); + !cpu_has_iowait(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_idle); u64 kcpustat_field_iowait(int cpu) { return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), ktime_get()); + cpu_has_iowait(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_iowait); #else @@ -627,7 +626,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), last_update_time); + !cpu_has_iowait(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -651,7 +650,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), last_update_time); + cpu_has_iowait(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); @@ -1136,7 +1135,6 @@ static int kcpustat_field_vtime(u64 *cpustat, int cpu, u64 *val) { struct vtime *vtime = &tsk->vtime; - struct rq *rq = cpu_rq(cpu); unsigned int seq; do { @@ -1179,11 +1177,11 @@ static int kcpustat_field_vtime(u64 *cpustat, *val += vtime->gtime + vtime_delta(vtime); break; case CPUTIME_IDLE: - if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait)) + if (state == VTIME_IDLE && !cpu_has_iowait(cpu)) *val += vtime_delta(vtime); break; case CPUTIME_IOWAIT: - if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0) + if (state == VTIME_IDLE && cpu_has_iowait(cpu)) *val += vtime_delta(vtime); break; default: @@ -1278,7 +1276,7 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, } break; case VTIME_IDLE: - if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0) + if (cpu_has_iowait(cpu)) cpustat[CPUTIME_IOWAIT] += delta; else cpustat[CPUTIME_IDLE] += delta;