diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..9db2d4fcead1 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,6 +80,7 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 591e3aa487fc..e2f3da71e03e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4124,6 +4124,28 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) return changed; } +/* + * Returns the number of CPUs available for this cgroup. + * + * This only really works for cgroup-v2 where all the controllers are mounted + * in the same hierarchy. If not cgroup-v2 or no cpuset controller is + * configured it reverts to num_online_cpus(). + */ +int cpuset_num_cpus(struct cgroup *cgrp) +{ + int nr = num_online_cpus(); + struct cpuset *cs; + + if (is_in_v2_mode()) { + guard(rcu)(); + cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); + if (cs) + nr = cpumask_weight(cs->effective_cpus); + } + + return nr; +} + void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 53b9e8218a2b..84e0ac40b6c0 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -640,6 +640,7 @@ static int cgroup_mode = 1; static const char *cgroup_mode_str[] = { "up", "smp", + "max", }; static int sched_cgroup_mode(const char *str) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3f8a2801f23e..b556d53a191c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4801,12 +4801,10 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); * * hence icky! */ -static long calc_smp_shares(struct cfs_rq *cfs_rq) +static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) { - long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - - tg_shares = READ_ONCE(tg->shares); + long tg_weight, load, shares; load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -4832,7 +4830,48 @@ static long calc_smp_shares(struct cfs_rq *cfs_rq) * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, tg_shares); + return clamp_t(long, shares, MIN_SHARES, shares_max); +} + +static int tg_cpus(struct task_group *tg) +{ + int nr = num_online_cpus(); + + if (cpusets_enabled()) { + struct cgroup *cgrp = tg->css.cgroup; + if (cgrp) + nr = cpuset_num_cpus(cgrp); + } + + return nr; +} + +/* + * Func: min(fraction(nr_cpus * tg->shares), nice -20) + * + * Scale tg->shares by the maximal number of CPUs; but clip the max shares at + * nice -20, otherwise a single spinner on a 512 CPU machine would result in + * 512*NICE_0_LOAD, which is also crazy. + */ +static long calc_max_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_cpus(tg); + long tg_shares = READ_ONCE(tg->shares); + long max_shares = scale_load(sched_prio_to_weight[0]); + return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); +} + +/* + * Func: fraction(tg->shares) + * + * This infamously results in tiny shares when you have many CPUs. + */ +static long calc_smp_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); } /* @@ -4857,6 +4896,9 @@ void __sched_cgroup_mode_update(int mode) default: func = &calc_smp_shares; break; + case 2: + func = &calc_max_shares; + break; } static_call_update(calc_group_shares, func); }