diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 84e0ac40b6c0..a5e3f8a365e6 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -640,6 +640,7 @@ static int cgroup_mode = 1; static const char *cgroup_mode_str[] = { "up", "smp", + "concur", "max", }; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b556d53a191c..64181e49b20f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4846,6 +4846,11 @@ static int tg_cpus(struct task_group *tg) return nr; } +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + /* * Func: min(fraction(nr_cpus * tg->shares), nice -20) * @@ -4862,6 +4867,20 @@ static long calc_max_shares(struct cfs_rq *cfs_rq) return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); } +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + /* * Func: fraction(tg->shares) * @@ -4897,6 +4916,9 @@ void __sched_cgroup_mode_update(int mode) func = &calc_smp_shares; break; case 2: + func = &calc_concur_shares; + break; + case 3: func = &calc_max_shares; break; } @@ -5043,7 +5065,7 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5064,17 +5086,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5084,9 +5110,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 922170b2bf93..fd267bfd51af 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,8 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -722,6 +724,7 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum;