github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 03:27:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge "sched/walt: expand cpu arrays"

Browse Source
This commit is contained in:
qctecmdr 2022-11-02 21:55:02 -07:00 committed by Gerrit - the friendly Code Review server
commit 010a200ec4
7 changed files with 556 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/sched/walt.h
View File

@ -20,7 +20,7 @@ enum pause_reason {
#if IS_ENABLED(CONFIG_SCHED_WALT)
#define MAX_CPUS_PER_CLUSTER 6
#define MAX_CLUSTERS 3
#define MAX_CLUSTERS 4
struct core_ctl_notif_data {
unsigned int nr_big;

502
kernel/sched/walt/core_ctl.c
View File

@ -16,6 +16,7 @@
#include <linux/syscore_ops.h>
#include <uapi/linux/sched/types.h>
#include <linux/sched/walt.h>
#include <linux/kstrtox.h>
#include "walt.h"
#include "trace.h"
@ -37,8 +38,8 @@ struct cluster_data {
unsigned int need_cpus;
unsigned int task_thres;
unsigned int max_nr;
unsigned int nr_prev_assist;
unsigned int nr_prev_assist_thresh;
unsigned int nr_assist;
unsigned int nr_assist_thresh;
s64 need_ts;
struct list_head lru;
bool enable;
@ -47,6 +48,10 @@ struct cluster_data {
unsigned int boost;
struct kobject kobj;
unsigned int strict_nrrun;
cpumask_t nrrun_cpu_mask;
cpumask_t nrrun_cpu_misfit_mask;
cpumask_t assist_cpu_mask;
cpumask_t assist_cpu_misfit_mask;
};
struct cpu_data {
@ -80,11 +85,14 @@ static DEFINE_SPINLOCK(state_lock);
static void apply_need(struct cluster_data *state);
static void wake_up_core_ctl_thread(void);
static bool initialized;
static bool assist_params_initialized;
ATOMIC_NOTIFIER_HEAD(core_ctl_notifier);
static unsigned int last_nr_big;
static unsigned int get_active_cpu_count(const struct cluster_data *cluster);
static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster);
static unsigned int get_active_count(cpumask_t *cpumask);
static void __ref do_core_ctl(void);
/* ========================= sysfs interface =========================== */
@ -163,13 +171,13 @@ static ssize_t store_task_thres(struct cluster_data *state,
return count;
}
static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state,
static ssize_t show_nr_assist_thresh(const struct cluster_data *state,
char *buf)
{
return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh);
return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_assist_thresh);
}
static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state,
static ssize_t store_nr_assist_thresh(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@ -177,7 +185,7 @@ static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state,
if (sscanf(buf, "%u\n", &val) != 1)
return -EINVAL;
state->nr_prev_assist_thresh = val;
state->nr_assist_thresh = val;
apply_need(state);
return count;
@ -399,6 +407,129 @@ static ssize_t show_not_preferred(const struct cluster_data *state, char *buf)
return count;
}
DECLARE_BITMAP(temp_bitmap, WALT_NR_CPUS);
static ssize_t store_nrrun_cpu_mask(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned long bitmask = 0xFF;
const unsigned long *bitmaskp = &bitmask;
unsigned long flags;
int ret = 0;
ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp);
if (ret < 0)
return ret;
bitmap_copy(temp_bitmap, bitmaskp, 8);
spin_lock_irqsave(&state_lock, flags);
cpumask_copy(&state->nrrun_cpu_mask, to_cpumask(temp_bitmap));
spin_unlock_irqrestore(&state_lock, flags);
return count;
}
static ssize_t show_nrrun_cpu_mask(const struct cluster_data *state, char *buf)
{
int ret = 0;
ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_mask))));
return ret;
}
static ssize_t store_nrrun_cpu_misfit_mask(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned long bitmask;
const unsigned long *bitmaskp = &bitmask;
unsigned long flags;
int ret = 0;
ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp);
if (ret < 0)
return ret;
bitmap_copy(temp_bitmap, bitmaskp, 8);
spin_lock_irqsave(&state_lock, flags);
cpumask_copy(&state->nrrun_cpu_misfit_mask, to_cpumask(temp_bitmap));
spin_unlock_irqrestore(&state_lock, flags);
return count;
}
static ssize_t show_nrrun_cpu_misfit_mask(const struct cluster_data *state, char *buf)
{
int ret = 0;
ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_misfit_mask))));
return ret;
}
static ssize_t store_assist_cpu_mask(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned long bitmask;
const unsigned long *bitmaskp = &bitmask;
unsigned long flags;
int ret = 0;
ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp);
if (ret < 0)
return ret;
bitmap_copy(temp_bitmap, bitmaskp, 8);
spin_lock_irqsave(&state_lock, flags);
cpumask_copy(&state->assist_cpu_mask, to_cpumask(temp_bitmap));
spin_unlock_irqrestore(&state_lock, flags);
return count;
}
static ssize_t show_assist_cpu_mask(const struct cluster_data *state, char *buf)
{
int ret = 0;
ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->assist_cpu_mask))));
return ret;
}
static ssize_t store_assist_cpu_misfit_mask(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned long bitmask;
const unsigned long *bitmaskp = &bitmask;
unsigned long flags;
int ret = 0;
ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp);
if (ret < 0)
return ret;
bitmap_copy(temp_bitmap, bitmaskp, 8);
spin_lock_irqsave(&state_lock, flags);
cpumask_copy(&state->assist_cpu_misfit_mask, to_cpumask(temp_bitmap));
spin_unlock_irqrestore(&state_lock, flags);
return count;
}
static ssize_t show_assist_cpu_misfit_mask(const struct cluster_data *state, char *buf)
{
int ret = 0;
ret = scnprintf(buf, PAGE_SIZE, "0x%x\n",
(*(cpumask_bits(&state->assist_cpu_misfit_mask))));
return ret;
}
struct core_ctl_attr {
struct attribute attr;
ssize_t (*show)(const struct cluster_data *cd, char *c);
@ -420,12 +551,16 @@ core_ctl_attr_rw(offline_delay_ms);
core_ctl_attr_rw(busy_up_thres);
core_ctl_attr_rw(busy_down_thres);
core_ctl_attr_rw(task_thres);
core_ctl_attr_rw(nr_prev_assist_thresh);
core_ctl_attr_rw(nr_assist_thresh);
core_ctl_attr_ro(need_cpus);
core_ctl_attr_ro(active_cpus);
core_ctl_attr_ro(global_state);
core_ctl_attr_rw(not_preferred);
core_ctl_attr_rw(enable);
core_ctl_attr_rw(nrrun_cpu_mask);
core_ctl_attr_rw(nrrun_cpu_misfit_mask);
core_ctl_attr_rw(assist_cpu_mask);
core_ctl_attr_rw(assist_cpu_misfit_mask);
static struct attribute *default_attrs[] = {
&min_cpus.attr,
@ -434,12 +569,16 @@ static struct attribute *default_attrs[] = {
&busy_up_thres.attr,
&busy_down_thres.attr,
&task_thres.attr,
&nr_prev_assist_thresh.attr,
&nr_assist_thresh.attr,
&enable.attr,
&need_cpus.attr,
&active_cpus.attr,
&global_state.attr,
&not_preferred.attr,
&nrrun_cpu_mask.attr,
&nrrun_cpu_misfit_mask.attr,
&assist_cpu_mask.attr,
&assist_cpu_misfit_mask.attr,
NULL
};
ATTRIBUTE_GROUPS(default);
@ -485,11 +624,16 @@ static struct kobj_type ktype_core_ctl = {
static struct sched_avg_stats *nr_stats;
/*
* nr_need:
/**
* compute_cluster_nr_run:
* @index: cluster index
*
* Number of tasks running on this cluster plus
* tasks running on higher capacity clusters.
* To find out CPUs needed from this cluster.
* tasks running on monitored clusters to find
* out CPUs needed from this cluster. Typically
* the other cpus that are monitored are from
* higher capacity clusters, and full clusters
* are considered.
*
* For example:
* On dual cluster system with 4 min capacity
@ -504,56 +648,51 @@ static struct sched_avg_stats *nr_stats;
* can be ready to accommodate tasks running on max
* capacity CPUs if the demand of tasks goes down.
*/
static int compute_cluster_nr_need(int index)
static int compute_cluster_nr_run(int index)
{
int cpu;
struct cluster_data *cluster;
int nr_need = 0;
for_each_cluster(cluster, index) {
for_each_cpu(cpu, &cluster->cpu_mask)
nr_need += nr_stats[cpu].nr;
}
cluster = &cluster_state[index];
for_each_cpu(cpu, &cluster->nrrun_cpu_mask)
nr_need += nr_stats[cpu].nr;
return nr_need;
}
/*
* prev_misfit_need:
* Tasks running on smaller capacity cluster which
* needs to be migrated to higher capacity cluster.
* To find out how many tasks need higher capacity CPUs.
*
* For example:
* On dual cluster system with 4 min capacity
* CPUs and 4 max capacity CPUs, if there are
* 2 small tasks and 2 big tasks running on
* min capacity CPUs and no tasks running on
* max cpacity, prev_misfit_need of min capacity
* cluster will be 0 and prev_misfit_need of
* max capacity cluster will be 2.
/**
* compute_cluster_nr_misfit:
* @index: cluster index
* Tasks running on cpus which this cluster is monitoring,
* that need to be migrated to this cluster. Typically,
* lower capacity cpus are monitored, and full clusters
* are considered.
*/
static int compute_prev_cluster_misfit_need(int index)
static int compute_cluster_nr_misfit(int index)
{
int cpu;
struct cluster_data *prev_cluster;
struct cluster_data *cluster;
int prev_misfit_need = 0;
/*
* Lowest capacity cluster does not have to
* accommodate any misfit tasks.
*/
if (index == 0)
return 0;
cluster = &cluster_state[index];
prev_cluster = &cluster_state[index - 1];
for_each_cpu(cpu, &prev_cluster->cpu_mask)
for_each_cpu(cpu, &cluster->nrrun_cpu_misfit_mask)
prev_misfit_need += nr_stats[cpu].nr_misfit;
return prev_misfit_need;
}
/**
* compute_cluster_max_nr
* @index: cluster index
*
* For each cpu in this cluster, determine the maximum
* number of tasks running on the cpu, and return the
* maximum number of tasks seen on a single cpu, for
* this cluster.
*/
static int compute_cluster_max_nr(int index)
{
int cpu;
@ -566,6 +705,105 @@ static int compute_cluster_max_nr(int index)
return max_nr;
}
/**
* cluster->nr_assist (aka prev_nr_need_assist) =
* compute_cluster_nr_run_assist() +
* compute_cluster_nr_run_misfit_assist() -
* get_assist_active_cpu_count()
*
* nr_assist is the number of tasks that are eligible to run on
* the monitored cpus, but cannot run because of insufficient
* CPUs there. The cpus being monitored yielding this information
* used by compute_cluster_nr_run_assist() and
* compute_cluster_nr_run_misfit_assist() are assist_cpu_mask and
* assist_cpu_misfit_mask, respectively.
*
* cluster->nr_assist is zero if there are no paused cpus in this cluster.
*
* For example:
*
* If max capacity cluster masks are defined as
* assist_cpu_mask=0x70 and assist_cpu_misfit_mask=0x0F:
*
* On tri-cluster system with 4 min capacity CPUs, 3 intermediate
* capacity CPUs and 1 max capacity CPU, if there are 4 small
* tasks running on min capacity CPUs, 4 big tasks running on
* intermediate capacity CPUs and no tasks running on max capacity
* CPU, nr_run_misfit_assist for min & max capacity clusters will be
* 0, but for intermediate capacity cluster nr_run_assist will be 1
* as it has 3 CPUs, but, there are 4 big tasks to be served.
*
* Since the max capacity cluster is monitoring intermediate
* clusters for number of tasks, and min capacity clusters
* for number of misfits and there are no misfits on min
* capacity cpus, this component is 0. Since there are 4 big
* tasks on intertmediate cap cpus but only 3 CPUs, nr_run_assist
* for the max capacity cluster is 1.
*
* As a further example, if the cluster had one misfit task in addition
* the small tasks on silvers, prime would have counted 1 task
* as a result of nr_run_misfit_assist CPUs, and 1 task as a result of
* nr_run_assist CPUs.
*
* In both cases the max capacity cpu is currently unpaused, nr_assist
* will be 0.
*/
/**
* compute_cluster_nr_run_assist:
* @index: cluster index
* Tasks running on cpus that this cluster
* is assisting. Typically the cpus being
* monitored are lower capacity cpus, not
* including the current cluster.
*/
static int compute_cluster_nr_run_assist(int index)
{
int cpu;
struct cluster_data *cluster = &cluster_state[index];
int nr_assist = 0;
for_each_cpu(cpu, &cluster->assist_cpu_mask)
nr_assist += nr_stats[cpu].nr;
return nr_assist;
}
/**
* compute_cluster_nr_run_assist:
* @index: cluster index
* Tasks running on cpus that this cluster
* is assisting for misfits. Typically the
* cpus being monitored are lower capacity
* cpus, not including the current cluster.
*
* In a 3 cluster system, this means that prime
* would be monitoring golds for assistance with
* misfits.
*/
static int compute_cluster_nr_misfit_assist(int index)
{
int cpu;
struct cluster_data *cluster = &cluster_state[index];
int nr_misfit_assist = 0;
for_each_cpu(cpu, &cluster->assist_cpu_misfit_mask)
nr_misfit_assist += nr_stats[cpu].nr;
return nr_misfit_assist;
}
/**
* cluster_real_big_tasks
* @index: cluster index
*
* Return the number of misfits on the lowest capacity
* cluster, or the number of tasks running on a bigger
* capacity cluster. This means that any task running
* on a non-min-capacity-cluster is considered a big
* task.
*/
static int cluster_real_big_tasks(int index)
{
int nr_big = 0;
@ -583,60 +821,6 @@ static int cluster_real_big_tasks(int index)
return nr_big;
}
/*
* prev_nr_need_assist:
* Tasks that are eligible to run on the previous
* cluster but cannot run because of insufficient
* CPUs there. prev_nr_need_assist is indicative
* of number of CPUs in this cluster that should
* assist its previous cluster to makeup for
* insufficient CPUs there.
*
* For example:
* On tri-cluster system with 4 min capacity
* CPUs, 3 intermediate capacity CPUs and 1
* max capacity CPU, if there are 4 small
* tasks running on min capacity CPUs, 4 big
* tasks running on intermediate capacity CPUs
* and no tasks running on max capacity CPU,
* prev_nr_need_assist for min & max capacity
* clusters will be 0, but, for intermediate
* capacity cluster prev_nr_need_assist will
* be 1 as it has 3 CPUs, but, there are 4 big
* tasks to be served.
*/
static int prev_cluster_nr_need_assist(int index)
{
int need = 0;
int cpu;
struct cluster_data *prev_cluster;
if (index == 0)
return 0;
index--;
prev_cluster = &cluster_state[index];
/*
* Next cluster should not assist, while there are paused cpus
* in this cluster.
*/
if (cluster_paused_cpus(prev_cluster))
return 0;
for_each_cpu(cpu, &prev_cluster->cpu_mask)
need += nr_stats[cpu].nr;
need += compute_prev_cluster_misfit_need(index);
if (need > prev_cluster->active_cpus)
need = need - prev_cluster->active_cpus;
else
need = 0;
return need;
}
/*
* This is only implemented for min capacity cluster.
*
@ -649,37 +833,39 @@ static int prev_cluster_nr_need_assist(int index)
* capacity CPUs from the nr and consider the remaining nr as
* strict and consider that many little CPUs are needed.
*/
static int compute_cluster_nr_strict_need(int index)
static int compute_cluster_strict_nr_run(int index)
{
int cpu;
struct cluster_data *cluster;
int nr_strict_need = 0;
int nr_scaled = 0;
if (index != 0)
return 0;
for_each_cluster(cluster, index) {
int nr_scaled = 0;
int active_cpus = cluster->active_cpus;
cluster = &cluster_state[index];
for_each_cpu(cpu, &cluster->cpu_mask)
nr_scaled += nr_stats[cpu].nr_scaled;
for_each_cpu(cpu, &cluster->nrrun_cpu_mask)
nr_scaled += nr_stats[cpu].nr_scaled;
nr_scaled /= 100;
nr_scaled /= 100;
/*
* For little cluster, nr_scaled becomes the nr_strict,
* for other cluster, overflow is counted towards
* the little cluster need.
*/
if (index == 0)
nr_strict_need += nr_scaled;
else
nr_strict_need += max(0, nr_scaled - active_cpus);
/*
* For little cluster, nr_scaled becomes the nr_strict,
* for other cluster, overflow is counted towards
* the little cluster need.
*/
if (index == 0) {
nr_strict_need += nr_scaled;
} else {
int active_cpus = get_active_count(&cluster->nrrun_cpu_mask);
nr_strict_need += max(0, nr_scaled - active_cpus);
}
return nr_strict_need;
}
static void update_running_avg(void)
{
struct cluster_data *cluster;
@ -691,24 +877,35 @@ static void update_running_avg(void)
spin_lock_irqsave(&state_lock, flags);
for_each_cluster(cluster, index) {
int nr_need, prev_misfit_need;
int nr_need, nr_misfit_need;
int nr_assist_need, nr_misfit_assist_need, nr_assist_active;
if (!cluster->inited)
continue;
nr_need = compute_cluster_nr_need(index);
prev_misfit_need = compute_prev_cluster_misfit_need(index);
nr_need = compute_cluster_nr_run(index);
nr_misfit_need = compute_cluster_nr_misfit(index);
cluster->nrrun = nr_need + prev_misfit_need;
cluster->nrrun = nr_need + nr_misfit_need;
cluster->max_nr = compute_cluster_max_nr(index);
cluster->nr_prev_assist = prev_cluster_nr_need_assist(index);
cluster->strict_nrrun = compute_cluster_nr_strict_need(index);
nr_assist_need = compute_cluster_nr_run_assist(index);
nr_misfit_assist_need = compute_cluster_nr_misfit_assist(index);
cluster->strict_nrrun = compute_cluster_strict_nr_run(index);
nr_assist_active = get_assist_active_cpu_count(cluster);
if (!cpumask_intersects(&cluster->assist_cpu_mask, &cpus_paused_by_us) &&
nr_assist_need + nr_misfit_assist_need > nr_assist_active)
cluster->nr_assist = nr_assist_need +
nr_misfit_assist_need - nr_assist_active;
else
cluster->nr_assist = 0;
trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need,
prev_misfit_need,
nr_misfit_need,
cluster->nrrun, cluster->max_nr,
cluster->nr_prev_assist);
cluster->nr_assist);
big_avg += cluster_real_big_tasks(index);
}
@ -731,8 +928,8 @@ static unsigned int apply_task_need(const struct cluster_data *cluster,
* resume as many cores as the previous cluster
* needs assistance with.
*/
if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh)
new_need = new_need + cluster->nr_prev_assist;
if (cluster->nr_assist >= cluster->nr_assist_thresh)
new_need = new_need + cluster->nr_assist;
/* only resume more cores if there are tasks to run */
if (cluster->nrrun > new_need)
@ -765,6 +962,14 @@ static unsigned int apply_limits(const struct cluster_data *cluster,
return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus);
}
static unsigned int get_active_count(cpumask_t *cpumask)
{
cpumask_t cpus;
cpumask_andnot(&cpus, cpumask, cpu_halt_mask);
return cpumask_weight(&cpus);
}
static unsigned int get_active_cpu_count(const struct cluster_data *cluster)
{
cpumask_t cpus;
@ -773,6 +978,14 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster)
return cpumask_weight(&cpus);
}
static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster)
{
cpumask_t cpus;
cpumask_andnot(&cpus, &cluster->assist_cpu_mask, cpu_halt_mask);
return cpumask_weight(&cpus);
}
static bool is_active(const struct cpu_data *state)
{
return cpu_active(state->cpu) && !cpu_halted(state->cpu);
@ -955,6 +1168,31 @@ static void core_ctl_call_notifier(void)
atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata);
}
/**
* core_ctl_check_masks_set
*
* return true if all clusters have updated values for their appropriate
* masks.
*/
static bool core_ctl_check_masks_set(void)
{
int index = 0;
struct cluster_data *cluster;
int possible_cpus = cpumask_weight(cpu_possible_mask);
int all_masks_set = true;
for_each_cluster(cluster, index) {
if (cpumask_weight(&cluster->nrrun_cpu_mask) > possible_cpus ||
cpumask_weight(&cluster->nrrun_cpu_misfit_mask) > possible_cpus ||
cpumask_weight(&cluster->assist_cpu_mask) > possible_cpus ||
cpumask_weight(&cluster->assist_cpu_misfit_mask) > possible_cpus) {
all_masks_set = false;
break;
}
}
return all_masks_set;
}
/*
* sched_get_nr_running_avg will wipe out previous statistics and
* update it to the values computed since the last call.
@ -975,6 +1213,11 @@ void core_ctl_check(u64 window_start)
if (unlikely(!initialized))
return;
if (unlikely(!assist_params_initialized)) {
assist_params_initialized = core_ctl_check_masks_set();
return;
}
if (window_start == core_ctl_check_timestamp)
return;
@ -1306,11 +1549,22 @@ static int cluster_init(const struct cpumask *mask)
cluster->need_cpus = cluster->num_cpus;
cluster->offline_delay_ms = 100;
cluster->task_thres = UINT_MAX;
cluster->nr_prev_assist_thresh = UINT_MAX;
cluster->nr_assist_thresh = UINT_MAX;
cluster->nrrun = cluster->num_cpus;
cluster->enable = true;
cluster->enable = false;
cluster->nr_not_preferred_cpus = 0;
cluster->strict_nrrun = 0;
/*
* set all cpus in the cluster. this is an invalid state
* and core control will not be considered initialized until
* this state is no longer true (all masks must be written).
*/
cpumask_setall(&cluster->nrrun_cpu_mask);
cpumask_setall(&cluster->nrrun_cpu_misfit_mask);
cpumask_setall(&cluster->assist_cpu_mask);
cpumask_setall(&cluster->assist_cpu_misfit_mask);
INIT_LIST_HEAD(&cluster->lru);
for_each_cpu(cpu, mask) {

56
kernel/sched/walt/sysctl.c
View File

@ -365,39 +365,25 @@ static void sched_update_updown_migrate_values(bool up)
{
int i = 0, cpu;
struct walt_sched_cluster *cluster;
int cap_margin_levels = num_sched_clusters - 1;
if (cap_margin_levels > 1) {
for_each_sched_cluster(cluster) {
/*
* No need to worry about CPUs in last cluster
* if there are more than 2 clusters in the system
*/
for_each_sched_cluster(cluster) {
for_each_cpu(cpu, &cluster->cpus) {
if (up)
sched_capacity_margin_up[cpu] =
SCHED_FIXEDPOINT_SCALE * 100 /
sysctl_sched_capacity_margin_up_pct[i];
else
sched_capacity_margin_down[cpu] =
SCHED_FIXEDPOINT_SCALE * 100 /
sysctl_sched_capacity_margin_dn_pct[i];
}
if (++i >= cap_margin_levels)
break;
}
} else {
for_each_possible_cpu(cpu) {
for_each_cpu(cpu, &cluster->cpus) {
if (up)
sched_capacity_margin_up[cpu] =
SCHED_FIXEDPOINT_SCALE * 100 /
sysctl_sched_capacity_margin_up_pct[0];
sysctl_sched_capacity_margin_up_pct[i];
else
sched_capacity_margin_down[cpu] =
sysctl_sched_capacity_margin_dn_pct[0];
SCHED_FIXEDPOINT_SCALE * 100 /
sysctl_sched_capacity_margin_dn_pct[i];
}
if (++i >= num_sched_clusters - 1)
break;
}
}
@ -468,6 +454,28 @@ int sched_updown_migrate_handler(struct ctl_table *table, int write,
return ret;
}
static void sched_update_updown_early_migrate_values(bool up)
{
int i = 0, cpu;
struct walt_sched_cluster *cluster;
for_each_sched_cluster(cluster) {
/*
* No need to worry about CPUs in last cluster
* if there are more than 2 clusters in the system
*/
for_each_cpu(cpu, &cluster->cpus) {
if (up)
sched_capacity_margin_early_up[cpu] = sysctl_sched_early_up[i];
else
sched_capacity_margin_early_down[cpu] = sysctl_sched_early_down[i];
}
if (++i >= num_sched_clusters - 1)
break;
}
}
int sched_updown_early_migrate_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos)
@ -504,7 +512,7 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write,
}
}
/* check up pct is greater than dn pct */
/* check up thresh is greater than dn thresh */
if (data == &sysctl_sched_early_up[0]) {
for (i = 0; i < cap_margin_levels; i++) {
if (val[i] >= sysctl_sched_early_down[i]) {
@ -525,6 +533,8 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write,
for (i = 0; i < cap_margin_levels; i++)
data[i] = val[i];
/* update individual cpu thresholds */
sched_update_updown_early_migrate_values(data == &sysctl_sched_early_up[0]);
unlock_mutex:
mutex_unlock(&mutex);

81
kernel/sched/walt/walt.c
View File

@ -44,8 +44,6 @@ const char *migrate_type_names[] = {
#define EARLY_DETECTION_DURATION 9500000
#define MAX_NUM_CGROUP_COLOC_ID 20
#define MAX_NR_CLUSTERS 3
#define NEW_TASK_ACTIVE_TIME 100000000
DEFINE_PER_CPU(struct walt_rq, walt_rq);
@ -2534,7 +2532,7 @@ static inline void assign_cluster_ids(struct list_head *head)
sched_cluster[pos++] = cluster;
}
WARN_ON(pos > MAX_NR_CLUSTERS);
WARN_ON(pos > MAX_CLUSTERS);
}
static inline void
@ -2585,16 +2583,39 @@ static void update_all_clusters_stats(void)
static bool walt_clusters_parsed;
cpumask_t __read_mostly **cpu_array;
u8 cpu_arrays_init_x11[1][1] = {
{0}, /* S */
};
u8 cpu_arrays_init_x22[2][2] = {
{0, 1}, /* S G */
{1, 0}, /* G S */
};
u8 cpu_arrays_init_x33[3][3] = {
{0, 1, 2}, /* S G P */
{1, 2, 0}, /* G P S */
{2, 1, 0}, /* P G S */
};
u8 cpu_arrays_init_x44[4][4] = {
{0, 2, 1, 3}, /* S T G P */
{1, 2, 3, 0}, /* G T P S */
{2, 3, 1, 0}, /* T P G S */
{3, 1, 2, 0}, /* P G T S */
};
static void init_cpu_array(void)
{
int i;
int rows = num_sched_clusters;
cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *),
cpu_array = kcalloc(rows, sizeof(cpumask_t *),
GFP_ATOMIC | __GFP_NOFAIL);
if (!cpu_array)
WALT_PANIC(1);
for (i = 0; i < num_sched_clusters; i++) {
for (i = 0; i < rows; i++) {
cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t),
GFP_ATOMIC | __GFP_NOFAIL);
if (!cpu_array[i])
@ -2604,35 +2625,35 @@ static void init_cpu_array(void)
static void build_cpu_array(void)
{
int i;
u8 *select_init_list;
u8 id;
int i, j;
if (!cpu_array)
WALT_PANIC(1);
/* Construct cpu_array row by row */
switch (num_sched_clusters) {
case 1:
select_init_list = (u8 *)cpu_arrays_init_x11;
break;
case 2:
select_init_list = (u8 *)cpu_arrays_init_x22;
break;
case 3:
select_init_list = (u8 *)cpu_arrays_init_x33;
break;
case 4:
select_init_list = (u8 *)cpu_arrays_init_x44;
break;
default:
pr_err("unsupported num clusters=%d\n", num_sched_clusters);
WALT_PANIC(1);
}
for (i = 0; i < num_sched_clusters; i++) {
int j, k = 1;
/* Fill out first column with appropriate cpu arrays */
cpumask_copy(&cpu_array[i][0], &sched_cluster[i]->cpus);
/*
* k starts from column 1 because 0 is filled
* Fill clusters for the rest of the row,
* above i in ascending order
*/
for (j = i + 1; j < num_sched_clusters; j++) {
cpumask_copy(&cpu_array[i][k],
&sched_cluster[j]->cpus);
k++;
}
/*
* k starts from where we left off above.
* Fill clusters below i in descending order.
*/
for (j = i - 1; j >= 0; j--) {
cpumask_copy(&cpu_array[i][k],
&sched_cluster[j]->cpus);
k++;
for (j = 0; j < num_sched_clusters; j++) {
id = select_init_list[i * num_sched_clusters + j];
cpumask_copy(&cpu_array[i][j], &sched_cluster[id]->cpus);
}
}
}

34
kernel/sched/walt/walt.h
View File

@ -770,51 +770,45 @@ static bool check_for_higher_capacity(int cpu1, int cpu2)
return capacity_orig_of(cpu1) > capacity_orig_of(cpu2);
}
extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS];
/* Migration margins for topapp */
extern unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS];
extern unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS];
static inline bool task_fits_capacity(struct task_struct *p,
long capacity,
int cpu)
int dst_cpu)
{
unsigned int margin;
unsigned long capacity = capacity_orig_of(dst_cpu);
/*
* Derive upmigration/downmigrate margin wrt the src/dest CPU.
*/
if (check_for_higher_capacity(task_cpu(p), cpu)) {
margin = sched_capacity_margin_down[cpu];
if (check_for_higher_capacity(task_cpu(p), dst_cpu)) {
margin = sched_capacity_margin_down[dst_cpu];
if (task_in_related_thread_group(p)) {
if (is_min_cluster_cpu(cpu))
margin = sysctl_sched_early_down[0];
else if (!is_max_cluster_cpu(cpu))
margin = sysctl_sched_early_down[1];
margin = sched_capacity_margin_early_down[dst_cpu];
}
} else {
margin = sched_capacity_margin_up[task_cpu(p)];
if (task_in_related_thread_group(p)) {
if (is_min_cluster_cpu(task_cpu(p)))
margin = sysctl_sched_early_up[0];
else if (!is_max_cluster_cpu(task_cpu(p)))
margin = sysctl_sched_early_up[1];
margin = sched_capacity_margin_early_up[task_cpu(p)];
}
}
return capacity * 1024 > uclamp_task_util(p) * margin;
}
static inline bool task_fits_max(struct task_struct *p, int cpu)
static inline bool task_fits_max(struct task_struct *p, int dst_cpu)
{
unsigned long capacity = capacity_orig_of(cpu);
unsigned long task_boost = per_task_boost(p);
if (is_max_cluster_cpu(cpu))
if (is_max_cluster_cpu(dst_cpu))
return true;
if (is_min_cluster_cpu(cpu)) {
if (is_min_cluster_cpu(dst_cpu)) {
if (task_boost_policy(p) == SCHED_BOOST_ON_BIG ||
task_boost > 0 ||
walt_uclamp_boosted(p) ||
walt_should_kick_upmigrate(p, cpu))
walt_should_kick_upmigrate(p, dst_cpu))
return false;
} else { /* mid cap cpu */
if (task_boost > TASK_BOOST_ON_MID)
@ -824,7 +818,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)
return true;
}
return task_fits_capacity(p, capacity, cpu);
return task_fits_capacity(p, dst_cpu);
}
extern struct sched_avg_stats *sched_get_nr_running_avg(void);

39
kernel/sched/walt/walt_cfs.c
View File

@ -61,6 +61,14 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = {
[0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */
};
/* Migration margins for topapp */
unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS] = {
[0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */
};
unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS] = {
[0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */
};
static inline bool
bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu)
{
@ -72,19 +80,17 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu)
return base_test && start_cap_test;
}
static inline bool task_demand_fits(struct task_struct *p, int cpu)
static inline bool task_demand_fits(struct task_struct *p, int dst_cpu)
{
unsigned long capacity = capacity_orig_of(cpu);
if (is_max_cluster_cpu(cpu))
if (is_max_cluster_cpu(dst_cpu))
return true;
if (!task_in_related_thread_group(p) && p->prio >= 124 &&
!is_min_cluster_cpu(cpu) && !is_max_cluster_cpu(cpu)) {
!is_min_cluster_cpu(dst_cpu) && !is_max_cluster_cpu(dst_cpu)) {
/* a non topapp low prio task fits on gold */
return true;
}
return task_fits_capacity(p, capacity, cpu);
return task_fits_capacity(p, dst_cpu);
}
struct find_best_target_env {
@ -165,8 +171,6 @@ static void walt_get_indicies(struct task_struct *p, int *order_index,
int *end_index, int per_task_boost, bool is_uclamp_boosted,
bool *energy_eval_needed)
{
int i = 0;
*order_index = 0;
*end_index = 0;
@ -182,9 +186,12 @@ static void walt_get_indicies(struct task_struct *p, int *order_index,
if (is_full_throttle_boost()) {
*energy_eval_needed = false;
*order_index = num_sched_clusters - 1;
if ((*order_index > 1) && task_demand_fits(p,
cpumask_first(&cpu_array[*order_index][1])))
*end_index = 1;
*end_index = num_sched_clusters - 2;
for (; *end_index >= 0; (*end_index)--)
if (task_demand_fits(p,
cpumask_first(&cpu_array[*order_index][*end_index])))
break;
return;
}
@ -193,19 +200,19 @@ static void walt_get_indicies(struct task_struct *p, int *order_index,
walt_task_skip_min_cpu(p)) {
*energy_eval_needed = false;
*order_index = 1;
*end_index = max(0, num_sched_clusters - 3);
if (sysctl_sched_asymcap_boost) {
*end_index = 1;
(*end_index)++;
return;
}
}
for (i = *order_index ; i < num_sched_clusters - 1; i++) {
if (task_demand_fits(p, cpumask_first(&cpu_array[i][0])))
for (; *order_index < num_sched_clusters - 1; (*order_index)++) {
if (task_demand_fits(p, cpumask_first(&cpu_array[*order_index][0])))
break;
}
*order_index = i;
if (*order_index == 0 &&
(task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) &&
!(p->in_iowait && task_in_related_thread_group(p)) &&

146
kernel/sched/walt/walt_lb.c
View File

@ -422,19 +422,34 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
/*
* find_first_idle_if_others_are_busy
*
* Get an idle cpu in the src_mask, iif the other cpus are busy
* with larger tasks or more than one task.
* Get an idle cpu in the middle clusters
*
* Returns -1 if there are no idle cpus in the mask, or if there
* is a CPU that is about to be come newly idle. Returns <cpu>
* if there is an idle cpu in a cluster that's not about to do
* newly idle load balancing.
* Also returns -1 if called on a single or dual cluster system
*/
static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask)
static int find_first_idle_if_others_are_busy(void)
{
int i, first_idle = -1;
struct cpumask src_mask;
cpumask_clear(&src_mask);
for (i = 0; i < num_sched_clusters; i++) {
if (i == 0 || i == num_sched_clusters - 1)
continue;
else
cpumask_or(&src_mask, &src_mask, &cpu_array[0][i]);
}
for_each_cpu(i, &src_mask) {
if (!cpu_active(i))
continue;
if (cpu_halted(i))
continue;
for_each_cpu(i, src_mask) {
if (available_idle_cpu(i))
first_idle = i;
@ -800,9 +815,10 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
int order_index;
int busy_cpu = -1;
bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD);
bool help_min_cap = false, find_next_cluster = false;
bool help_min_cap = false;
int first_idle;
int has_misfit = 0;
int i;
if (unlikely(walt_disabled))
return;
@ -858,98 +874,48 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
* can be queued remotely, so keep a check on nr_running
* and bail out.
*/
if (num_sched_clusters <= 2) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
if (num_sched_clusters == 2) {
has_misfit = false;
find_next_cluster = (order_index == 0) ? enough_idle : 1;
if (find_next_cluster) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu,
&cpu_array[order_index][1], &has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
}
}
order_index = wrq->cluster->id;
for (i = 0; i < num_sched_clusters; i++) {
int first_cpu = cpumask_first(&cpu_array[order_index][i]);
struct walt_rq *src_wrq = &per_cpu(walt_rq, first_cpu);
int src_cluster_id = src_wrq->cluster->id;
goto unlock;
}
if (order_index == 0) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
if (enough_idle) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
}
/*
* help the farthest cluster by kicking an idle cpu in the next
* cluster. In case no idle is found, pull it in.
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][i],
&has_misfit, true);
if (busy_cpu == -1)
continue;
/* when not enough idle
* Small should not help big.
* Big should help small ONLY is mifit is present.
* Same capacity cpus should help each other
*/
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit, true);
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
if (!enough_idle &&
(capacity_orig_of(this_cpu) < capacity_orig_of(busy_cpu) ||
(capacity_orig_of(this_cpu) > capacity_orig_of(busy_cpu) && !has_misfit)))
continue;
/* if helping farthest cluster, kick a middle */
if (num_sched_clusters > 2 &&
((wrq->cluster->id == 0 && src_cluster_id == num_sched_clusters - 1) ||
(wrq->cluster->id == num_sched_clusters - 1 && src_cluster_id == 0))) {
first_idle = find_first_idle_if_others_are_busy();
if (first_idle != -1) {
walt_kick_cpu(first_idle);
} else if (walt_rotation_enabled) {
goto found_busy_cpu;
} else {
if (walt_rotation_enabled &&
capacity_orig_of(this_cpu) >
capacity_orig_of(busy_cpu)) {
/*
* When BTR active help
* smallest immediately
*/
goto found_busy_cpu;
}
}
} else {
goto found_busy_cpu;
}
} else if (order_index == 2) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
/* help gold only if prime has had enough idle or gold has a misfit */
has_misfit = false;
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
/* help the farthest cluster indirectly if it needs help */
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit, true);
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
if (first_idle != -1) {
walt_kick_cpu(first_idle);
} else if (walt_rotation_enabled) {
goto found_busy_cpu;
}
}
} else {
busy_cpu =
walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
if (enough_idle) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
}
has_misfit = false;
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
}
goto unlock;