From 5c0b58bd1c9f09c4e9363665702c82694c6ee7a0 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 12 Mar 2026 16:04:18 +0100 Subject: [PATCH] sched/fair: Add cgroup_mode: concur Improve upon the previous scheme ("max") by no longer assuming maximal concurrency. Instead scale by: 'min(nr_tasks, nr_cpus)'. This handles the low concurrency cases more gracefully: F_g_n' = min(M, N) * F_g_n Notably this is the first mode where: avg(F_g_n) = 1 In the single task case it reduces to ("smp") and then it nicely scales up until it hits N, where it behaves like ("max"). This is no longer clipped at nice -20. Strictly speaking it isn't different from the normal SMP scenario where all tasks are extremely unbalanced. There are no unnatural inflations in this scheme. The meaning of "cpu.weight" would be: weight per active CPU. NOTE: Compute the group wide number of tasks by extending the tg->load_avg computation with tg->runnable_avg, since cfs_rq->runnable_avg is based on cfs_rq->h_nr_running. Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260605124051.740585993%40infradead.org --- kernel/sched/debug.c | 1 + kernel/sched/fair.c | 43 ++++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 3 +++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 84e0ac40b6c0..a5e3f8a365e6 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -640,6 +640,7 @@ static int cgroup_mode = 1; static const char *cgroup_mode_str[] = { "up", "smp", + "concur", "max", }; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b556d53a191c..64181e49b20f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4846,6 +4846,11 @@ static int tg_cpus(struct task_group *tg) return nr; } +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + /* * Func: min(fraction(nr_cpus * tg->shares), nice -20) * @@ -4862,6 +4867,20 @@ static long calc_max_shares(struct cfs_rq *cfs_rq) return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); } +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + /* * Func: fraction(tg->shares) * @@ -4897,6 +4916,9 @@ void __sched_cgroup_mode_update(int mode) func = &calc_smp_shares; break; case 2: + func = &calc_concur_shares; + break; + case 3: func = &calc_max_shares; break; } @@ -5043,7 +5065,7 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5064,17 +5086,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5084,9 +5110,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 922170b2bf93..fd267bfd51af 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,8 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -722,6 +724,7 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum;