github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-30 11:11:26 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/walt/core_ctl: create cpu masks for core control

Browse Source 
Core Control programatically determins its behavior, in terms
of what other clusters this particular cluster would pay attention
to, to decide whether to unpause a cpu. This is inflexible, and
prevents an easy path to utilizing more clustersto allow unpausing
of cpus to assist clusters in unique ways to support power and
performance goals.

In preparation for using 4 clusters with core control, increase
the flexibility of core control in terms of how it chooses which
clusters to assist, for this cluster.

With this change there are 4 classes of cpus each cluster pays
attention to. Cpus being monitored for nrrun, nrrun misfits,
assist, and assist missfits. This allows the code to replicate
previous core control behavior, while introducing the ability
to specify any cpu to monitor for each of these behaviors.

For example, a typical configuration for a typical 3 cluster
system would be as below, replicating previous core control
behavior but using these masks.

                   nrrun             assist
 cluster | nrrun | misfit | assist | misfit |
 --------------------------------------------
 silver  | 0xFF  | 0x00   | 0x00   | 0x00   |
 --------------------------------------------
 gold    | 0xF8  | 0x07   | 0x00   | 0x00   |
 --------------------------------------------
 prime   | 0x80  | 0x78   | 0x78   | 0x07   |
 --------------------------------------------

This means, for example, when prime computes nrrun it will only
consider itself, for nrrun misfits, golds, and assisting, golds,
and for assisting with misfits, silvers.

This gets expanded to a 4 entry table for a 4 cluster system as
the clusters are fed into core control initialization, allowing
for an extremely flexible definition of how core control operates.

Change-Id: Ic372da9a453ef593144172524ae1d99e056dd45f
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>
Signed-off-by: Abhijeet Dharmapurikar <quic_adharmap@quicinc.com>
This commit is contained in:
Stephen Dickey 2022-09-29 13:28:31 -07:00
parent 2b0465b6c3
commit f1d0a9ba42
1 changed files with 204 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

328
kernel/sched/walt/core_ctl.c
View File

@ -37,8 +37,8 @@ struct cluster_data {
unsigned int need_cpus;
unsigned int task_thres;
unsigned int max_nr;
unsigned int nr_prev_assist;
unsigned int nr_prev_assist_thresh;
unsigned int nr_assist;
unsigned int nr_assist_thresh;
s64 need_ts;
struct list_head lru;
bool enable;
@ -47,6 +47,10 @@ struct cluster_data {
unsigned int boost;
struct kobject kobj;
unsigned int strict_nrrun;
cpumask_t nrrun_cpu_mask;
cpumask_t nrrun_cpu_misfit_mask;
cpumask_t assist_cpu_mask;
cpumask_t assist_cpu_misfit_mask;
};
struct cpu_data {
@ -85,6 +89,8 @@ ATOMIC_NOTIFIER_HEAD(core_ctl_notifier);
static unsigned int last_nr_big;
static unsigned int get_active_cpu_count(const struct cluster_data *cluster);
static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster);
static unsigned int get_active_count(cpumask_t *cpumask);
static void __ref do_core_ctl(void);
/* ========================= sysfs interface =========================== */
@ -163,13 +169,13 @@ static ssize_t store_task_thres(struct cluster_data *state,
return count;
}
static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state,
static ssize_t show_nr_assist_thresh(const struct cluster_data *state,
char *buf)
{
return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh);
return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_assist_thresh);
}
static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state,
static ssize_t store_nr_assist_thresh(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@ -177,7 +183,7 @@ static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state,
if (sscanf(buf, "%u\n", &val) != 1)
return -EINVAL;
state->nr_prev_assist_thresh = val;
state->nr_assist_thresh = val;
apply_need(state);
return count;
@ -420,7 +426,7 @@ core_ctl_attr_rw(offline_delay_ms);
core_ctl_attr_rw(busy_up_thres);
core_ctl_attr_rw(busy_down_thres);
core_ctl_attr_rw(task_thres);
core_ctl_attr_rw(nr_prev_assist_thresh);
core_ctl_attr_rw(nr_assist_thresh);
core_ctl_attr_ro(need_cpus);
core_ctl_attr_ro(active_cpus);
core_ctl_attr_ro(global_state);
@ -434,7 +440,7 @@ static struct attribute *default_attrs[] = {
&busy_up_thres.attr,
&busy_down_thres.attr,
&task_thres.attr,
&nr_prev_assist_thresh.attr,
&nr_assist_thresh.attr,
&enable.attr,
&need_cpus.attr,
&active_cpus.attr,
@ -485,11 +491,16 @@ static struct kobj_type ktype_core_ctl = {
static struct sched_avg_stats *nr_stats;
/*
* nr_need:
/**
* compute_cluster_nr_run:
* @index: cluster index
*
* Number of tasks running on this cluster plus
* tasks running on higher capacity clusters.
* To find out CPUs needed from this cluster.
* tasks running on monitored clusters to find
* out CPUs needed from this cluster. Typically
* the other cpus that are monitored are from
* higher capacity clusters, and full clusters
* are considered.
*
* For example:
* On dual cluster system with 4 min capacity
@ -504,56 +515,51 @@ static struct sched_avg_stats *nr_stats;
* can be ready to accommodate tasks running on max
* capacity CPUs if the demand of tasks goes down.
*/
static int compute_cluster_nr_need(int index)
static int compute_cluster_nr_run(int index)
{
int cpu;
struct cluster_data *cluster;
int nr_need = 0;
for_each_cluster(cluster, index) {
for_each_cpu(cpu, &cluster->cpu_mask)
nr_need += nr_stats[cpu].nr;
}
cluster = &cluster_state[index];
for_each_cpu(cpu, &cluster->nrrun_cpu_mask)
nr_need += nr_stats[cpu].nr;
return nr_need;
}
/*
* prev_misfit_need:
* Tasks running on smaller capacity cluster which
* needs to be migrated to higher capacity cluster.
* To find out how many tasks need higher capacity CPUs.
*
* For example:
* On dual cluster system with 4 min capacity
* CPUs and 4 max capacity CPUs, if there are
* 2 small tasks and 2 big tasks running on
* min capacity CPUs and no tasks running on
* max cpacity, prev_misfit_need of min capacity
* cluster will be 0 and prev_misfit_need of
* max capacity cluster will be 2.
/**
* compute_cluster_nr_misfit:
* @index: cluster index
* Tasks running on cpus which this cluster is monitoring,
* that need to be migrated to this cluster. Typically,
* lower capacity cpus are monitored, and full clusters
* are considered.
*/
static int compute_prev_cluster_misfit_need(int index)
static int compute_cluster_nr_misfit(int index)
{
int cpu;
struct cluster_data *prev_cluster;
struct cluster_data *cluster;
int prev_misfit_need = 0;
/*
* Lowest capacity cluster does not have to
* accommodate any misfit tasks.
*/
if (index == 0)
return 0;
cluster = &cluster_state[index];
prev_cluster = &cluster_state[index - 1];
for_each_cpu(cpu, &prev_cluster->cpu_mask)
for_each_cpu(cpu, &cluster->nrrun_cpu_misfit_mask)
prev_misfit_need += nr_stats[cpu].nr_misfit;
return prev_misfit_need;
}
/**
* compute_cluster_max_nr
* @index: cluster index
*
* For each cpu in this cluster, determine the maximum
* number of tasks running on the cpu, and return the
* maximum number of tasks seen on a single cpu, for
* this cluster.
*/
static int compute_cluster_max_nr(int index)
{
int cpu;
@ -566,6 +572,105 @@ static int compute_cluster_max_nr(int index)
return max_nr;
}
/**
* cluster->nr_assist (aka prev_nr_need_assist) =
* compute_cluster_nr_run_assist() +
* compute_cluster_nr_run_misfit_assist() -
* get_assist_active_cpu_count()
*
* nr_assist is the number of tasks that are eligible to run on
* the monitored cpus, but cannot run because of insufficient
* CPUs there. The cpus being monitored yielding this information
* used by compute_cluster_nr_run_assist() and
* compute_cluster_nr_run_misfit_assist() are assist_cpu_mask and
* assist_cpu_misfit_mask, respectively.
*
* cluster->nr_assist is zero if there are no paused cpus in this cluster.
*
* For example:
*
* If max capacity cluster masks are defined as
* assist_cpu_mask=0x70 and assist_cpu_misfit_mask=0x0F:
*
* On tri-cluster system with 4 min capacity CPUs, 3 intermediate
* capacity CPUs and 1 max capacity CPU, if there are 4 small
* tasks running on min capacity CPUs, 4 big tasks running on
* intermediate capacity CPUs and no tasks running on max capacity
* CPU, nr_run_misfit_assist for min & max capacity clusters will be
* 0, but for intermediate capacity cluster nr_run_assist will be 1
* as it has 3 CPUs, but, there are 4 big tasks to be served.
*
* Since the max capacity cluster is monitoring intermediate
* clusters for number of tasks, and min capacity clusters
* for number of misfits and there are no misfits on min
* capacity cpus, this component is 0. Since there are 4 big
* tasks on intertmediate cap cpus but only 3 CPUs, nr_run_assist
* for the max capacity cluster is 1.
*
* As a further example, if the cluster had one misfit task in addition
* the small tasks on silvers, prime would have counted 1 task
* as a result of nr_run_misfit_assist CPUs, and 1 task as a result of
* nr_run_assist CPUs.
*
* In both cases the max capacity cpu is currently unpaused, nr_assist
* will be 0.
*/
/**
* compute_cluster_nr_run_assist:
* @index: cluster index
* Tasks running on cpus that this cluster
* is assisting. Typically the cpus being
* monitored are lower capacity cpus, not
* including the current cluster.
*/
static int compute_cluster_nr_run_assist(int index)
{
int cpu;
struct cluster_data *cluster = &cluster_state[index];
int nr_assist = 0;
for_each_cpu(cpu, &cluster->assist_cpu_mask)
nr_assist += nr_stats[cpu].nr;
return nr_assist;
}
/**
* compute_cluster_nr_run_assist:
* @index: cluster index
* Tasks running on cpus that this cluster
* is assisting for misfits. Typically the
* cpus being monitored are lower capacity
* cpus, not including the current cluster.
*
* In a 3 cluster system, this means that prime
* would be monitoring golds for assistance with
* misfits.
*/
static int compute_cluster_nr_misfit_assist(int index)
{
int cpu;
struct cluster_data *cluster = &cluster_state[index];
int nr_misfit_assist = 0;
for_each_cpu(cpu, &cluster->assist_cpu_misfit_mask)
nr_misfit_assist += nr_stats[cpu].nr;
return nr_misfit_assist;
}
/**
* cluster_real_big_tasks
* @index: cluster index
*
* Return the number of misfits on the lowest capacity
* cluster, or the number of tasks running on a bigger
* capacity cluster. This means that any task running
* on a non-min-capacity-cluster is considered a big
* task.
*/
static int cluster_real_big_tasks(int index)
{
int nr_big = 0;
@ -583,60 +688,6 @@ static int cluster_real_big_tasks(int index)
return nr_big;
}
/*
* prev_nr_need_assist:
* Tasks that are eligible to run on the previous
* cluster but cannot run because of insufficient
* CPUs there. prev_nr_need_assist is indicative
* of number of CPUs in this cluster that should
* assist its previous cluster to makeup for
* insufficient CPUs there.
*
* For example:
* On tri-cluster system with 4 min capacity
* CPUs, 3 intermediate capacity CPUs and 1
* max capacity CPU, if there are 4 small
* tasks running on min capacity CPUs, 4 big
* tasks running on intermediate capacity CPUs
* and no tasks running on max capacity CPU,
* prev_nr_need_assist for min & max capacity
* clusters will be 0, but, for intermediate
* capacity cluster prev_nr_need_assist will
* be 1 as it has 3 CPUs, but, there are 4 big
* tasks to be served.
*/
static int prev_cluster_nr_need_assist(int index)
{
int need = 0;
int cpu;
struct cluster_data *prev_cluster;
if (index == 0)
return 0;
index--;
prev_cluster = &cluster_state[index];
/*
* Next cluster should not assist, while there are paused cpus
* in this cluster.
*/
if (cluster_paused_cpus(prev_cluster))
return 0;
for_each_cpu(cpu, &prev_cluster->cpu_mask)
need += nr_stats[cpu].nr;
need += compute_prev_cluster_misfit_need(index);
if (need > prev_cluster->active_cpus)
need = need - prev_cluster->active_cpus;
else
need = 0;
return need;
}
/*
* This is only implemented for min capacity cluster.
*
@ -649,37 +700,39 @@ static int prev_cluster_nr_need_assist(int index)
* capacity CPUs from the nr and consider the remaining nr as
* strict and consider that many little CPUs are needed.
*/
static int compute_cluster_nr_strict_need(int index)
static int compute_cluster_strict_nr_run(int index)
{
int cpu;
struct cluster_data *cluster;
int nr_strict_need = 0;
int nr_scaled = 0;
if (index != 0)
return 0;
for_each_cluster(cluster, index) {
int nr_scaled = 0;
int active_cpus = cluster->active_cpus;
cluster = &cluster_state[index];
for_each_cpu(cpu, &cluster->cpu_mask)
nr_scaled += nr_stats[cpu].nr_scaled;
for_each_cpu(cpu, &cluster->nrrun_cpu_mask)
nr_scaled += nr_stats[cpu].nr_scaled;
nr_scaled /= 100;
nr_scaled /= 100;
/*
* For little cluster, nr_scaled becomes the nr_strict,
* for other cluster, overflow is counted towards
* the little cluster need.
*/
if (index == 0)
nr_strict_need += nr_scaled;
else
nr_strict_need += max(0, nr_scaled - active_cpus);
/*
* For little cluster, nr_scaled becomes the nr_strict,
* for other cluster, overflow is counted towards
* the little cluster need.
*/
if (index == 0) {
nr_strict_need += nr_scaled;
} else {
int active_cpus = get_active_count(&cluster->nrrun_cpu_mask);
nr_strict_need += max(0, nr_scaled - active_cpus);
}
return nr_strict_need;
}
static void update_running_avg(void)
{
struct cluster_data *cluster;
@ -691,24 +744,35 @@ static void update_running_avg(void)
spin_lock_irqsave(&state_lock, flags);
for_each_cluster(cluster, index) {
int nr_need, prev_misfit_need;
int nr_need, nr_misfit_need;
int nr_assist_need, nr_misfit_assist_need, nr_assist_active;
if (!cluster->inited)
continue;
nr_need = compute_cluster_nr_need(index);
prev_misfit_need = compute_prev_cluster_misfit_need(index);
nr_need = compute_cluster_nr_run(index);
nr_misfit_need = compute_cluster_nr_misfit(index);
cluster->nrrun = nr_need + prev_misfit_need;
cluster->nrrun = nr_need + nr_misfit_need;
cluster->max_nr = compute_cluster_max_nr(index);
cluster->nr_prev_assist = prev_cluster_nr_need_assist(index);
cluster->strict_nrrun = compute_cluster_nr_strict_need(index);
nr_assist_need = compute_cluster_nr_run_assist(index);
nr_misfit_assist_need = compute_cluster_nr_misfit_assist(index);
cluster->strict_nrrun = compute_cluster_strict_nr_run(index);
nr_assist_active = get_assist_active_cpu_count(cluster);
if (!cpumask_intersects(&cluster->assist_cpu_mask, &cpus_paused_by_us) &&
nr_assist_need + nr_misfit_assist_need > nr_assist_active)
cluster->nr_assist = nr_assist_need +
nr_misfit_assist_need - nr_assist_active;
else
cluster->nr_assist = 0;
trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need,
prev_misfit_need,
nr_misfit_need,
cluster->nrrun, cluster->max_nr,
cluster->nr_prev_assist);
cluster->nr_assist);
big_avg += cluster_real_big_tasks(index);
}
@ -731,8 +795,8 @@ static unsigned int apply_task_need(const struct cluster_data *cluster,
* resume as many cores as the previous cluster
* needs assistance with.
*/
if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh)
new_need = new_need + cluster->nr_prev_assist;
if (cluster->nr_assist >= cluster->nr_assist_thresh)
new_need = new_need + cluster->nr_assist;
/* only resume more cores if there are tasks to run */
if (cluster->nrrun > new_need)
@ -765,6 +829,14 @@ static unsigned int apply_limits(const struct cluster_data *cluster,
return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus);
}
static unsigned int get_active_count(cpumask_t *cpumask)
{
cpumask_t cpus;
cpumask_andnot(&cpus, cpumask, cpu_halt_mask);
return cpumask_weight(&cpus);
}
static unsigned int get_active_cpu_count(const struct cluster_data *cluster)
{
cpumask_t cpus;
@ -773,6 +845,14 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster)
return cpumask_weight(&cpus);
}
static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster)
{
cpumask_t cpus;
cpumask_andnot(&cpus, &cluster->assist_cpu_mask, cpu_halt_mask);
return cpumask_weight(&cpus);
}
static bool is_active(const struct cpu_data *state)
{
return cpu_active(state->cpu) && !cpu_halted(state->cpu);
@ -1306,9 +1386,9 @@ static int cluster_init(const struct cpumask *mask)
cluster->need_cpus = cluster->num_cpus;
cluster->offline_delay_ms = 100;
cluster->task_thres = UINT_MAX;
cluster->nr_prev_assist_thresh = UINT_MAX;
cluster->nr_assist_thresh = UINT_MAX;
cluster->nrrun = cluster->num_cpus;
cluster->enable = true;
cluster->enable = false;
cluster->nr_not_preferred_cpus = 0;
cluster->strict_nrrun = 0;
INIT_LIST_HEAD(&cluster->lru);