From f1d0a9ba426145fcdc1554c05a679143cbdb77a8 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 29 Sep 2022 13:28:31 -0700 Subject: [PATCH 01/10] sched/walt/core_ctl: create cpu masks for core control Core Control programatically determins its behavior, in terms of what other clusters this particular cluster would pay attention to, to decide whether to unpause a cpu. This is inflexible, and prevents an easy path to utilizing more clustersto allow unpausing of cpus to assist clusters in unique ways to support power and performance goals. In preparation for using 4 clusters with core control, increase the flexibility of core control in terms of how it chooses which clusters to assist, for this cluster. With this change there are 4 classes of cpus each cluster pays attention to. Cpus being monitored for nrrun, nrrun misfits, assist, and assist missfits. This allows the code to replicate previous core control behavior, while introducing the ability to specify any cpu to monitor for each of these behaviors. For example, a typical configuration for a typical 3 cluster system would be as below, replicating previous core control behavior but using these masks. nrrun assist cluster | nrrun | misfit | assist | misfit | -------------------------------------------- silver | 0xFF | 0x00 | 0x00 | 0x00 | -------------------------------------------- gold | 0xF8 | 0x07 | 0x00 | 0x00 | -------------------------------------------- prime | 0x80 | 0x78 | 0x78 | 0x07 | -------------------------------------------- This means, for example, when prime computes nrrun it will only consider itself, for nrrun misfits, golds, and assisting, golds, and for assisting with misfits, silvers. This gets expanded to a 4 entry table for a 4 cluster system as the clusters are fed into core control initialization, allowing for an extremely flexible definition of how core control operates. Change-Id: Ic372da9a453ef593144172524ae1d99e056dd45f Signed-off-by: Stephen Dickey Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/core_ctl.c | 328 ++++++++++++++++++++++------------- 1 file changed, 204 insertions(+), 124 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index a7dee202065e..44a5e40e3bdd 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -37,8 +37,8 @@ struct cluster_data { unsigned int need_cpus; unsigned int task_thres; unsigned int max_nr; - unsigned int nr_prev_assist; - unsigned int nr_prev_assist_thresh; + unsigned int nr_assist; + unsigned int nr_assist_thresh; s64 need_ts; struct list_head lru; bool enable; @@ -47,6 +47,10 @@ struct cluster_data { unsigned int boost; struct kobject kobj; unsigned int strict_nrrun; + cpumask_t nrrun_cpu_mask; + cpumask_t nrrun_cpu_misfit_mask; + cpumask_t assist_cpu_mask; + cpumask_t assist_cpu_misfit_mask; }; struct cpu_data { @@ -85,6 +89,8 @@ ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); static unsigned int last_nr_big; static unsigned int get_active_cpu_count(const struct cluster_data *cluster); +static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster); +static unsigned int get_active_count(cpumask_t *cpumask); static void __ref do_core_ctl(void); /* ========================= sysfs interface =========================== */ @@ -163,13 +169,13 @@ static ssize_t store_task_thres(struct cluster_data *state, return count; } -static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state, +static ssize_t show_nr_assist_thresh(const struct cluster_data *state, char *buf) { - return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh); + return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_assist_thresh); } -static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, +static ssize_t store_nr_assist_thresh(struct cluster_data *state, const char *buf, size_t count) { unsigned int val; @@ -177,7 +183,7 @@ static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, if (sscanf(buf, "%u\n", &val) != 1) return -EINVAL; - state->nr_prev_assist_thresh = val; + state->nr_assist_thresh = val; apply_need(state); return count; @@ -420,7 +426,7 @@ core_ctl_attr_rw(offline_delay_ms); core_ctl_attr_rw(busy_up_thres); core_ctl_attr_rw(busy_down_thres); core_ctl_attr_rw(task_thres); -core_ctl_attr_rw(nr_prev_assist_thresh); +core_ctl_attr_rw(nr_assist_thresh); core_ctl_attr_ro(need_cpus); core_ctl_attr_ro(active_cpus); core_ctl_attr_ro(global_state); @@ -434,7 +440,7 @@ static struct attribute *default_attrs[] = { &busy_up_thres.attr, &busy_down_thres.attr, &task_thres.attr, - &nr_prev_assist_thresh.attr, + &nr_assist_thresh.attr, &enable.attr, &need_cpus.attr, &active_cpus.attr, @@ -485,11 +491,16 @@ static struct kobj_type ktype_core_ctl = { static struct sched_avg_stats *nr_stats; -/* - * nr_need: +/** + * compute_cluster_nr_run: + * @index: cluster index + * * Number of tasks running on this cluster plus - * tasks running on higher capacity clusters. - * To find out CPUs needed from this cluster. + * tasks running on monitored clusters to find + * out CPUs needed from this cluster. Typically + * the other cpus that are monitored are from + * higher capacity clusters, and full clusters + * are considered. * * For example: * On dual cluster system with 4 min capacity @@ -504,56 +515,51 @@ static struct sched_avg_stats *nr_stats; * can be ready to accommodate tasks running on max * capacity CPUs if the demand of tasks goes down. */ -static int compute_cluster_nr_need(int index) +static int compute_cluster_nr_run(int index) { int cpu; struct cluster_data *cluster; int nr_need = 0; - for_each_cluster(cluster, index) { - for_each_cpu(cpu, &cluster->cpu_mask) - nr_need += nr_stats[cpu].nr; - } + cluster = &cluster_state[index]; + + for_each_cpu(cpu, &cluster->nrrun_cpu_mask) + nr_need += nr_stats[cpu].nr; return nr_need; } -/* - * prev_misfit_need: - * Tasks running on smaller capacity cluster which - * needs to be migrated to higher capacity cluster. - * To find out how many tasks need higher capacity CPUs. - * - * For example: - * On dual cluster system with 4 min capacity - * CPUs and 4 max capacity CPUs, if there are - * 2 small tasks and 2 big tasks running on - * min capacity CPUs and no tasks running on - * max cpacity, prev_misfit_need of min capacity - * cluster will be 0 and prev_misfit_need of - * max capacity cluster will be 2. +/** + * compute_cluster_nr_misfit: + * @index: cluster index + * Tasks running on cpus which this cluster is monitoring, + * that need to be migrated to this cluster. Typically, + * lower capacity cpus are monitored, and full clusters + * are considered. */ -static int compute_prev_cluster_misfit_need(int index) +static int compute_cluster_nr_misfit(int index) { int cpu; - struct cluster_data *prev_cluster; + struct cluster_data *cluster; int prev_misfit_need = 0; - /* - * Lowest capacity cluster does not have to - * accommodate any misfit tasks. - */ - if (index == 0) - return 0; + cluster = &cluster_state[index]; - prev_cluster = &cluster_state[index - 1]; - - for_each_cpu(cpu, &prev_cluster->cpu_mask) + for_each_cpu(cpu, &cluster->nrrun_cpu_misfit_mask) prev_misfit_need += nr_stats[cpu].nr_misfit; return prev_misfit_need; } +/** + * compute_cluster_max_nr + * @index: cluster index + * + * For each cpu in this cluster, determine the maximum + * number of tasks running on the cpu, and return the + * maximum number of tasks seen on a single cpu, for + * this cluster. + */ static int compute_cluster_max_nr(int index) { int cpu; @@ -566,6 +572,105 @@ static int compute_cluster_max_nr(int index) return max_nr; } +/** + * cluster->nr_assist (aka prev_nr_need_assist) = + * compute_cluster_nr_run_assist() + + * compute_cluster_nr_run_misfit_assist() - + * get_assist_active_cpu_count() + * + * nr_assist is the number of tasks that are eligible to run on + * the monitored cpus, but cannot run because of insufficient + * CPUs there. The cpus being monitored yielding this information + * used by compute_cluster_nr_run_assist() and + * compute_cluster_nr_run_misfit_assist() are assist_cpu_mask and + * assist_cpu_misfit_mask, respectively. + * + * cluster->nr_assist is zero if there are no paused cpus in this cluster. + * + * For example: + * + * If max capacity cluster masks are defined as + * assist_cpu_mask=0x70 and assist_cpu_misfit_mask=0x0F: + * + * On tri-cluster system with 4 min capacity CPUs, 3 intermediate + * capacity CPUs and 1 max capacity CPU, if there are 4 small + * tasks running on min capacity CPUs, 4 big tasks running on + * intermediate capacity CPUs and no tasks running on max capacity + * CPU, nr_run_misfit_assist for min & max capacity clusters will be + * 0, but for intermediate capacity cluster nr_run_assist will be 1 + * as it has 3 CPUs, but, there are 4 big tasks to be served. + * + * Since the max capacity cluster is monitoring intermediate + * clusters for number of tasks, and min capacity clusters + * for number of misfits and there are no misfits on min + * capacity cpus, this component is 0. Since there are 4 big + * tasks on intertmediate cap cpus but only 3 CPUs, nr_run_assist + * for the max capacity cluster is 1. + * + * As a further example, if the cluster had one misfit task in addition + * the small tasks on silvers, prime would have counted 1 task + * as a result of nr_run_misfit_assist CPUs, and 1 task as a result of + * nr_run_assist CPUs. + * + * In both cases the max capacity cpu is currently unpaused, nr_assist + * will be 0. + */ + +/** + * compute_cluster_nr_run_assist: + * @index: cluster index + * Tasks running on cpus that this cluster + * is assisting. Typically the cpus being + * monitored are lower capacity cpus, not + * including the current cluster. + */ +static int compute_cluster_nr_run_assist(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int nr_assist = 0; + + for_each_cpu(cpu, &cluster->assist_cpu_mask) + nr_assist += nr_stats[cpu].nr; + + return nr_assist; +} + +/** + * compute_cluster_nr_run_assist: + * @index: cluster index + * Tasks running on cpus that this cluster + * is assisting for misfits. Typically the + * cpus being monitored are lower capacity + * cpus, not including the current cluster. + * + * In a 3 cluster system, this means that prime + * would be monitoring golds for assistance with + * misfits. + */ +static int compute_cluster_nr_misfit_assist(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int nr_misfit_assist = 0; + + for_each_cpu(cpu, &cluster->assist_cpu_misfit_mask) + nr_misfit_assist += nr_stats[cpu].nr; + + return nr_misfit_assist; +} + + +/** + * cluster_real_big_tasks + * @index: cluster index + * + * Return the number of misfits on the lowest capacity + * cluster, or the number of tasks running on a bigger + * capacity cluster. This means that any task running + * on a non-min-capacity-cluster is considered a big + * task. + */ static int cluster_real_big_tasks(int index) { int nr_big = 0; @@ -583,60 +688,6 @@ static int cluster_real_big_tasks(int index) return nr_big; } -/* - * prev_nr_need_assist: - * Tasks that are eligible to run on the previous - * cluster but cannot run because of insufficient - * CPUs there. prev_nr_need_assist is indicative - * of number of CPUs in this cluster that should - * assist its previous cluster to makeup for - * insufficient CPUs there. - * - * For example: - * On tri-cluster system with 4 min capacity - * CPUs, 3 intermediate capacity CPUs and 1 - * max capacity CPU, if there are 4 small - * tasks running on min capacity CPUs, 4 big - * tasks running on intermediate capacity CPUs - * and no tasks running on max capacity CPU, - * prev_nr_need_assist for min & max capacity - * clusters will be 0, but, for intermediate - * capacity cluster prev_nr_need_assist will - * be 1 as it has 3 CPUs, but, there are 4 big - * tasks to be served. - */ -static int prev_cluster_nr_need_assist(int index) -{ - int need = 0; - int cpu; - struct cluster_data *prev_cluster; - - if (index == 0) - return 0; - - index--; - prev_cluster = &cluster_state[index]; - - /* - * Next cluster should not assist, while there are paused cpus - * in this cluster. - */ - if (cluster_paused_cpus(prev_cluster)) - return 0; - - for_each_cpu(cpu, &prev_cluster->cpu_mask) - need += nr_stats[cpu].nr; - - need += compute_prev_cluster_misfit_need(index); - - if (need > prev_cluster->active_cpus) - need = need - prev_cluster->active_cpus; - else - need = 0; - - return need; -} - /* * This is only implemented for min capacity cluster. * @@ -649,37 +700,39 @@ static int prev_cluster_nr_need_assist(int index) * capacity CPUs from the nr and consider the remaining nr as * strict and consider that many little CPUs are needed. */ -static int compute_cluster_nr_strict_need(int index) +static int compute_cluster_strict_nr_run(int index) { int cpu; struct cluster_data *cluster; int nr_strict_need = 0; + int nr_scaled = 0; if (index != 0) return 0; - for_each_cluster(cluster, index) { - int nr_scaled = 0; - int active_cpus = cluster->active_cpus; + cluster = &cluster_state[index]; - for_each_cpu(cpu, &cluster->cpu_mask) - nr_scaled += nr_stats[cpu].nr_scaled; + for_each_cpu(cpu, &cluster->nrrun_cpu_mask) + nr_scaled += nr_stats[cpu].nr_scaled; - nr_scaled /= 100; + nr_scaled /= 100; - /* - * For little cluster, nr_scaled becomes the nr_strict, - * for other cluster, overflow is counted towards - * the little cluster need. - */ - if (index == 0) - nr_strict_need += nr_scaled; - else - nr_strict_need += max(0, nr_scaled - active_cpus); + /* + * For little cluster, nr_scaled becomes the nr_strict, + * for other cluster, overflow is counted towards + * the little cluster need. + */ + if (index == 0) { + nr_strict_need += nr_scaled; + } else { + int active_cpus = get_active_count(&cluster->nrrun_cpu_mask); + + nr_strict_need += max(0, nr_scaled - active_cpus); } return nr_strict_need; } + static void update_running_avg(void) { struct cluster_data *cluster; @@ -691,24 +744,35 @@ static void update_running_avg(void) spin_lock_irqsave(&state_lock, flags); for_each_cluster(cluster, index) { - int nr_need, prev_misfit_need; + int nr_need, nr_misfit_need; + int nr_assist_need, nr_misfit_assist_need, nr_assist_active; if (!cluster->inited) continue; - nr_need = compute_cluster_nr_need(index); - prev_misfit_need = compute_prev_cluster_misfit_need(index); + nr_need = compute_cluster_nr_run(index); + nr_misfit_need = compute_cluster_nr_misfit(index); - cluster->nrrun = nr_need + prev_misfit_need; + cluster->nrrun = nr_need + nr_misfit_need; cluster->max_nr = compute_cluster_max_nr(index); - cluster->nr_prev_assist = prev_cluster_nr_need_assist(index); - cluster->strict_nrrun = compute_cluster_nr_strict_need(index); + nr_assist_need = compute_cluster_nr_run_assist(index); + nr_misfit_assist_need = compute_cluster_nr_misfit_assist(index); + + cluster->strict_nrrun = compute_cluster_strict_nr_run(index); + nr_assist_active = get_assist_active_cpu_count(cluster); + + if (!cpumask_intersects(&cluster->assist_cpu_mask, &cpus_paused_by_us) && + nr_assist_need + nr_misfit_assist_need > nr_assist_active) + cluster->nr_assist = nr_assist_need + + nr_misfit_assist_need - nr_assist_active; + else + cluster->nr_assist = 0; trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need, - prev_misfit_need, + nr_misfit_need, cluster->nrrun, cluster->max_nr, - cluster->nr_prev_assist); + cluster->nr_assist); big_avg += cluster_real_big_tasks(index); } @@ -731,8 +795,8 @@ static unsigned int apply_task_need(const struct cluster_data *cluster, * resume as many cores as the previous cluster * needs assistance with. */ - if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh) - new_need = new_need + cluster->nr_prev_assist; + if (cluster->nr_assist >= cluster->nr_assist_thresh) + new_need = new_need + cluster->nr_assist; /* only resume more cores if there are tasks to run */ if (cluster->nrrun > new_need) @@ -765,6 +829,14 @@ static unsigned int apply_limits(const struct cluster_data *cluster, return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus); } +static unsigned int get_active_count(cpumask_t *cpumask) +{ + cpumask_t cpus; + + cpumask_andnot(&cpus, cpumask, cpu_halt_mask); + return cpumask_weight(&cpus); +} + static unsigned int get_active_cpu_count(const struct cluster_data *cluster) { cpumask_t cpus; @@ -773,6 +845,14 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster) return cpumask_weight(&cpus); } +static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster) +{ + cpumask_t cpus; + + cpumask_andnot(&cpus, &cluster->assist_cpu_mask, cpu_halt_mask); + return cpumask_weight(&cpus); +} + static bool is_active(const struct cpu_data *state) { return cpu_active(state->cpu) && !cpu_halted(state->cpu); @@ -1306,9 +1386,9 @@ static int cluster_init(const struct cpumask *mask) cluster->need_cpus = cluster->num_cpus; cluster->offline_delay_ms = 100; cluster->task_thres = UINT_MAX; - cluster->nr_prev_assist_thresh = UINT_MAX; + cluster->nr_assist_thresh = UINT_MAX; cluster->nrrun = cluster->num_cpus; - cluster->enable = true; + cluster->enable = false; cluster->nr_not_preferred_cpus = 0; cluster->strict_nrrun = 0; INIT_LIST_HEAD(&cluster->lru); From aff07c50a1d1ca12fee7b068875eeececdc12fce Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 30 Sep 2022 15:41:39 -0700 Subject: [PATCH 02/10] sched/walt/core_ctl: make the cpumasks writable The user should be able to determine how this particular soc has setup the cpumasks for controlling core control. Make those visible and allow them to be written by postboot. Change-Id: Icb74ab9573dbe392481032a8a460ba6938bf287a Signed-off-by: Stephen Dickey Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/core_ctl.c | 132 +++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 44a5e40e3bdd..0840bed754ac 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "walt.h" #include "trace.h" @@ -405,6 +406,129 @@ static ssize_t show_not_preferred(const struct cluster_data *state, char *buf) return count; } +DECLARE_BITMAP(temp_bitmap, WALT_NR_CPUS); + +static ssize_t store_nrrun_cpu_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask = 0xFF; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->nrrun_cpu_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_nrrun_cpu_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_mask)))); + + return ret; +} + +static ssize_t store_nrrun_cpu_misfit_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->nrrun_cpu_misfit_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_nrrun_cpu_misfit_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_misfit_mask)))); + + return ret; +} + +static ssize_t store_assist_cpu_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->assist_cpu_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_assist_cpu_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->assist_cpu_mask)))); + + return ret; +} + +static ssize_t store_assist_cpu_misfit_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->assist_cpu_misfit_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_assist_cpu_misfit_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", + (*(cpumask_bits(&state->assist_cpu_misfit_mask)))); + + return ret; +} + struct core_ctl_attr { struct attribute attr; ssize_t (*show)(const struct cluster_data *cd, char *c); @@ -432,6 +556,10 @@ core_ctl_attr_ro(active_cpus); core_ctl_attr_ro(global_state); core_ctl_attr_rw(not_preferred); core_ctl_attr_rw(enable); +core_ctl_attr_rw(nrrun_cpu_mask); +core_ctl_attr_rw(nrrun_cpu_misfit_mask); +core_ctl_attr_rw(assist_cpu_mask); +core_ctl_attr_rw(assist_cpu_misfit_mask); static struct attribute *default_attrs[] = { &min_cpus.attr, @@ -446,6 +574,10 @@ static struct attribute *default_attrs[] = { &active_cpus.attr, &global_state.attr, ¬_preferred.attr, + &nrrun_cpu_mask.attr, + &nrrun_cpu_misfit_mask.attr, + &assist_cpu_mask.attr, + &assist_cpu_misfit_mask.attr, NULL }; ATTRIBUTE_GROUPS(default); From 10120e444abf22cf407931b00bdc5febdb2aad8b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 11 Oct 2022 13:37:57 -0700 Subject: [PATCH 03/10] sched/walt/core_ctl: wait for masks initialization The core control masks must be initialized before core control can be used. Prepare core control to validate the masks before allowing core control check to proceed. Change-Id: I86e7a51a009ba03347ecb0814b0122247bd54d74 Signed-off-by: Stephen Dickey Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/core_ctl.c | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 0840bed754ac..21726f06ba9f 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -85,6 +85,7 @@ static DEFINE_SPINLOCK(state_lock); static void apply_need(struct cluster_data *state); static void wake_up_core_ctl_thread(void); static bool initialized; +static bool assist_params_initialized; ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); static unsigned int last_nr_big; @@ -1167,6 +1168,31 @@ static void core_ctl_call_notifier(void) atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata); } +/** + * core_ctl_check_masks_set + * + * return true if all clusters have updated values for their appropriate + * masks. + */ +static bool core_ctl_check_masks_set(void) +{ + int index = 0; + struct cluster_data *cluster; + int possible_cpus = cpumask_weight(cpu_possible_mask); + int all_masks_set = true; + + for_each_cluster(cluster, index) { + if (cpumask_weight(&cluster->nrrun_cpu_mask) > possible_cpus || + cpumask_weight(&cluster->nrrun_cpu_misfit_mask) > possible_cpus || + cpumask_weight(&cluster->assist_cpu_mask) > possible_cpus || + cpumask_weight(&cluster->assist_cpu_misfit_mask) > possible_cpus) { + all_masks_set = false; + break; + } + } + + return all_masks_set; +} /* * sched_get_nr_running_avg will wipe out previous statistics and * update it to the values computed since the last call. @@ -1187,6 +1213,11 @@ void core_ctl_check(u64 window_start) if (unlikely(!initialized)) return; + if (unlikely(!assist_params_initialized)) { + assist_params_initialized = core_ctl_check_masks_set(); + return; + } + if (window_start == core_ctl_check_timestamp) return; @@ -1523,6 +1554,17 @@ static int cluster_init(const struct cpumask *mask) cluster->enable = false; cluster->nr_not_preferred_cpus = 0; cluster->strict_nrrun = 0; + + /* + * set all cpus in the cluster. this is an invalid state + * and core control will not be considered initialized until + * this state is no longer true (all masks must be written). + */ + cpumask_setall(&cluster->nrrun_cpu_mask); + cpumask_setall(&cluster->nrrun_cpu_misfit_mask); + cpumask_setall(&cluster->assist_cpu_mask); + cpumask_setall(&cluster->assist_cpu_misfit_mask); + INIT_LIST_HEAD(&cluster->lru); for_each_cpu(cpu, mask) { From 9003596e0b47aec0c61f246d4bade922b5d1b096 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 29 Sep 2022 11:45:35 -0700 Subject: [PATCH 04/10] sched/walt: Update MAX_CLUSTERS to 4 In preparation of a 4 cluster system, indicate our maximum number of clusters as 4. Change-Id: I06e097581d737b56f2cbdc20460264801e29ffa2 Signed-off-by: Stephen Dickey Signed-off-by: Abhijeet Dharmapurikar --- include/linux/sched/walt.h | 2 +- kernel/sched/walt/walt.c | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 8a213e937820..56d0ea018597 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -20,7 +20,7 @@ enum pause_reason { #if IS_ENABLED(CONFIG_SCHED_WALT) #define MAX_CPUS_PER_CLUSTER 6 -#define MAX_CLUSTERS 3 +#define MAX_CLUSTERS 4 struct core_ctl_notif_data { unsigned int nr_big; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 23f9a01f9a26..2788c9323459 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -43,8 +43,6 @@ const char *migrate_type_names[] = { #define EARLY_DETECTION_DURATION 9500000 #define MAX_NUM_CGROUP_COLOC_ID 20 -#define MAX_NR_CLUSTERS 3 - #define NEW_TASK_ACTIVE_TIME 100000000 DEFINE_PER_CPU(struct walt_rq, walt_rq); @@ -2533,7 +2531,7 @@ static inline void assign_cluster_ids(struct list_head *head) sched_cluster[pos++] = cluster; } - WARN_ON(pos > MAX_NR_CLUSTERS); + WARN_ON(pos > MAX_CLUSTERS); } static inline void From a569117648ab796a817af54d2a4f84bed011b253 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 22 Sep 2022 13:09:40 -0700 Subject: [PATCH 05/10] sched/walt: update walt_get_indicies for 4 clusters walt_get_indices() was written for a 3 cluster system. In the future it is necessary to support 2, 3 and 4 cluster systems. Make the appropriate changes to walt_get_indices such that cpus are visited in the correct order for each of these topologies. Change-Id: Iae60c152aaa561326adccc0e92056e598dde2679 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 456f5560743b..83183cd8c873 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -165,8 +165,6 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, int *end_index, int per_task_boost, bool is_uclamp_boosted, bool *energy_eval_needed) { - int i = 0; - *order_index = 0; *end_index = 0; @@ -182,9 +180,12 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, if (is_full_throttle_boost()) { *energy_eval_needed = false; *order_index = num_sched_clusters - 1; - if ((*order_index > 1) && task_demand_fits(p, - cpumask_first(&cpu_array[*order_index][1]))) - *end_index = 1; + *end_index = num_sched_clusters - 2; + + for (; *end_index >= 0; (*end_index)--) + if (task_demand_fits(p, + cpumask_first(&cpu_array[*order_index][*end_index]))) + break; return; } @@ -193,19 +194,19 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, walt_task_skip_min_cpu(p)) { *energy_eval_needed = false; *order_index = 1; + *end_index = max(0, num_sched_clusters - 3); + if (sysctl_sched_asymcap_boost) { - *end_index = 1; + (*end_index)++; return; } } - for (i = *order_index ; i < num_sched_clusters - 1; i++) { - if (task_demand_fits(p, cpumask_first(&cpu_array[i][0]))) + for (; *order_index < num_sched_clusters - 1; (*order_index)++) { + if (task_demand_fits(p, cpumask_first(&cpu_array[*order_index][0]))) break; } - *order_index = i; - if (*order_index == 0 && (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && !(p->in_iowait && task_in_related_thread_group(p)) && From 0b76f7d59db50ed9d220cebd435a313a19f9d01a Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 19 Oct 2022 15:04:16 -0700 Subject: [PATCH 06/10] sched/walt: don't pass capacity to task_fits_max There is no need for the callsites to pass the capacity of the new cpu. It is best if task_fits_max() gets it as it is the only one that uses it. Also take this opportunity to change the variable name "cpu" to "dst_cpu". Change-Id: I836f779b31a4c8f7bc5f9c87c40f4fa5caf786f2 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.h | 23 +++++++++++------------ kernel/sched/walt/walt_cfs.c | 10 ++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index c72e68120ee5..37182af1feb0 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -772,20 +772,20 @@ static bool check_for_higher_capacity(int cpu1, int cpu2) extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; static inline bool task_fits_capacity(struct task_struct *p, - long capacity, - int cpu) + int dst_cpu) { unsigned int margin; + unsigned long capacity = capacity_orig_of(dst_cpu); /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. */ - if (check_for_higher_capacity(task_cpu(p), cpu)) { - margin = sched_capacity_margin_down[cpu]; + if (check_for_higher_capacity(task_cpu(p), dst_cpu)) { + margin = sched_capacity_margin_down[dst_cpu]; if (task_in_related_thread_group(p)) { - if (is_min_cluster_cpu(cpu)) + if (is_min_cluster_cpu(dst_cpu)) margin = sysctl_sched_early_down[0]; - else if (!is_max_cluster_cpu(cpu)) + else if (!is_max_cluster_cpu(dst_cpu)) margin = sysctl_sched_early_down[1]; } } else { @@ -801,19 +801,18 @@ static inline bool task_fits_capacity(struct task_struct *p, return capacity * 1024 > uclamp_task_util(p) * margin; } -static inline bool task_fits_max(struct task_struct *p, int cpu) +static inline bool task_fits_max(struct task_struct *p, int dst_cpu) { - unsigned long capacity = capacity_orig_of(cpu); unsigned long task_boost = per_task_boost(p); - if (is_max_cluster_cpu(cpu)) + if (is_max_cluster_cpu(dst_cpu)) return true; - if (is_min_cluster_cpu(cpu)) { + if (is_min_cluster_cpu(dst_cpu)) { if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || task_boost > 0 || walt_uclamp_boosted(p) || - walt_should_kick_upmigrate(p, cpu)) + walt_should_kick_upmigrate(p, dst_cpu)) return false; } else { /* mid cap cpu */ if (task_boost > TASK_BOOST_ON_MID) @@ -823,7 +822,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) return true; } - return task_fits_capacity(p, capacity, cpu); + return task_fits_capacity(p, dst_cpu); } extern struct sched_avg_stats *sched_get_nr_running_avg(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 83183cd8c873..e143c013c7e4 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -72,19 +72,17 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) return base_test && start_cap_test; } -static inline bool task_demand_fits(struct task_struct *p, int cpu) +static inline bool task_demand_fits(struct task_struct *p, int dst_cpu) { - unsigned long capacity = capacity_orig_of(cpu); - - if (is_max_cluster_cpu(cpu)) + if (is_max_cluster_cpu(dst_cpu)) return true; if (!task_in_related_thread_group(p) && p->prio >= 124 && - !is_min_cluster_cpu(cpu) && !is_max_cluster_cpu(cpu)) { + !is_min_cluster_cpu(dst_cpu) && !is_max_cluster_cpu(dst_cpu)) { /* a non topapp low prio task fits on gold */ return true; } - return task_fits_capacity(p, capacity, cpu); + return task_fits_capacity(p, dst_cpu); } struct find_best_target_env { From 87f1fd9d3458b6e7a18dcacf685561dd8a5a9a03 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 22 Sep 2022 15:00:47 -0700 Subject: [PATCH 07/10] sched/walt: fix handling of up/down migrate The code currently unnecessarily checks for cap_margin_levels > 1. In SMP systems where num_sched_clusters = 1 we still use a single sched_cluster with all the cpus. So we dont have to handle num_sched_cluster =1 (or 2) separately. Change-Id: Ie9a8dda70555f02d64df11b310fd56727f149633 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/sysctl.c | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 38b0c731eae4..896ee8fd6759 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -365,39 +365,25 @@ static void sched_update_updown_migrate_values(bool up) { int i = 0, cpu; struct walt_sched_cluster *cluster; - int cap_margin_levels = num_sched_clusters - 1; - if (cap_margin_levels > 1) { + for_each_sched_cluster(cluster) { /* * No need to worry about CPUs in last cluster * if there are more than 2 clusters in the system */ - for_each_sched_cluster(cluster) { - for_each_cpu(cpu, &cluster->cpus) { - if (up) - sched_capacity_margin_up[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_up_pct[i]; - else - sched_capacity_margin_down[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_dn_pct[i]; - } - - if (++i >= cap_margin_levels) - break; - } - } else { - for_each_possible_cpu(cpu) { + for_each_cpu(cpu, &cluster->cpus) { if (up) sched_capacity_margin_up[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_up_pct[0]; + sysctl_sched_capacity_margin_up_pct[i]; else sched_capacity_margin_down[cpu] = - sysctl_sched_capacity_margin_dn_pct[0]; + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_dn_pct[i]; } + + if (++i >= num_sched_clusters - 1) + break; } } From cd5ecacb5b3db89fdd8d243787741b402c7091c5 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 22 Sep 2022 15:42:38 -0700 Subject: [PATCH 08/10] sched/walt: use individual cpu early up/down threshold Currently the code is using up/down thresholds based on clusters. With number of clusters changing, it will be best if we populate a per cpu value and use it. Change-Id: Id16aef6f9cecb485fefe4d4e56f2b07852aa6e1c Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/sysctl.c | 26 +++++++++++++++++++++++++- kernel/sched/walt/walt.h | 15 +++++---------- kernel/sched/walt/walt_cfs.c | 8 ++++++++ 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 896ee8fd6759..4762798ab7ee 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -454,6 +454,28 @@ int sched_updown_migrate_handler(struct ctl_table *table, int write, return ret; } +static void sched_update_updown_early_migrate_values(bool up) +{ + int i = 0, cpu; + struct walt_sched_cluster *cluster; + + for_each_sched_cluster(cluster) { + /* + * No need to worry about CPUs in last cluster + * if there are more than 2 clusters in the system + */ + for_each_cpu(cpu, &cluster->cpus) { + if (up) + sched_capacity_margin_early_up[cpu] = sysctl_sched_early_up[i]; + else + sched_capacity_margin_early_down[cpu] = sysctl_sched_early_down[i]; + } + + if (++i >= num_sched_clusters - 1) + break; + } +} + int sched_updown_early_migrate_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) @@ -490,7 +512,7 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write, } } - /* check up pct is greater than dn pct */ + /* check up thresh is greater than dn thresh */ if (data == &sysctl_sched_early_up[0]) { for (i = 0; i < cap_margin_levels; i++) { if (val[i] >= sysctl_sched_early_down[i]) { @@ -511,6 +533,8 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write, for (i = 0; i < cap_margin_levels; i++) data[i] = val[i]; + /* update individual cpu thresholds */ + sched_update_updown_early_migrate_values(data == &sysctl_sched_early_up[0]); unlock_mutex: mutex_unlock(&mutex); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 37182af1feb0..3954933e7990 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -769,8 +769,9 @@ static bool check_for_higher_capacity(int cpu1, int cpu2) return capacity_orig_of(cpu1) > capacity_orig_of(cpu2); } -extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; -extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; +/* Migration margins for topapp */ +extern unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS]; +extern unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS]; static inline bool task_fits_capacity(struct task_struct *p, int dst_cpu) { @@ -783,18 +784,12 @@ static inline bool task_fits_capacity(struct task_struct *p, if (check_for_higher_capacity(task_cpu(p), dst_cpu)) { margin = sched_capacity_margin_down[dst_cpu]; if (task_in_related_thread_group(p)) { - if (is_min_cluster_cpu(dst_cpu)) - margin = sysctl_sched_early_down[0]; - else if (!is_max_cluster_cpu(dst_cpu)) - margin = sysctl_sched_early_down[1]; + margin = sched_capacity_margin_early_down[dst_cpu]; } } else { margin = sched_capacity_margin_up[task_cpu(p)]; if (task_in_related_thread_group(p)) { - if (is_min_cluster_cpu(task_cpu(p))) - margin = sysctl_sched_early_up[0]; - else if (!is_max_cluster_cpu(task_cpu(p))) - margin = sysctl_sched_early_up[1]; + margin = sched_capacity_margin_early_up[task_cpu(p)]; } } diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index e143c013c7e4..e88ba99b7bde 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -61,6 +61,14 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ }; +/* Migration margins for topapp */ +unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */ +}; +unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ +}; + static inline bool bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { From eb867a1f36dc36b903508b8d9ec0c7f84cae5e23 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 27 Sep 2022 15:07:07 -0700 Subject: [PATCH 09/10] sched/walt: update load balancer for multiple cluster Currently the code it setup for specific number of clusters. Make it generic for N clusters. Change-Id: I6ad1c068e8e5645afd61e111783d981381a710b2 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 146 ++++++++++++++---------------------- 1 file changed, 56 insertions(+), 90 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index be8027a21d54..8b031c87a4f8 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -422,19 +422,34 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) /* * find_first_idle_if_others_are_busy * - * Get an idle cpu in the src_mask, iif the other cpus are busy - * with larger tasks or more than one task. + * Get an idle cpu in the middle clusters * * Returns -1 if there are no idle cpus in the mask, or if there * is a CPU that is about to be come newly idle. Returns * if there is an idle cpu in a cluster that's not about to do * newly idle load balancing. + * Also returns -1 if called on a single or dual cluster system */ -static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) +static int find_first_idle_if_others_are_busy(void) { int i, first_idle = -1; + struct cpumask src_mask; + + cpumask_clear(&src_mask); + for (i = 0; i < num_sched_clusters; i++) { + if (i == 0 || i == num_sched_clusters - 1) + continue; + else + cpumask_or(&src_mask, &src_mask, &cpu_array[0][i]); + } + + for_each_cpu(i, &src_mask) { + if (!cpu_active(i)) + continue; + + if (cpu_halted(i)) + continue; - for_each_cpu(i, src_mask) { if (available_idle_cpu(i)) first_idle = i; @@ -800,9 +815,10 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int order_index; int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); - bool help_min_cap = false, find_next_cluster = false; + bool help_min_cap = false; int first_idle; int has_misfit = 0; + int i; if (unlikely(walt_disabled)) return; @@ -858,98 +874,48 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * can be queued remotely, so keep a check on nr_running * and bail out. */ - if (num_sched_clusters <= 2) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - if (num_sched_clusters == 2) { - has_misfit = false; - find_next_cluster = (order_index == 0) ? enough_idle : 1; - if (find_next_cluster) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, - &cpu_array[order_index][1], &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; - } - } + order_index = wrq->cluster->id; + for (i = 0; i < num_sched_clusters; i++) { + int first_cpu = cpumask_first(&cpu_array[order_index][i]); + struct walt_rq *src_wrq = &per_cpu(walt_rq, first_cpu); + int src_cluster_id = src_wrq->cluster->id; - goto unlock; - } - - if (order_index == 0) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - if (enough_idle) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - } - - /* - * help the farthest cluster by kicking an idle cpu in the next - * cluster. In case no idle is found, pull it in. + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][i], + &has_misfit, true); + if (busy_cpu == -1) + continue; + /* when not enough idle + * Small should not help big. + * Big should help small ONLY is mifit is present. + * Same capacity cpus should help each other */ - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1) { - first_idle = - find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + if (!enough_idle && + (capacity_orig_of(this_cpu) < capacity_orig_of(busy_cpu) || + (capacity_orig_of(this_cpu) > capacity_orig_of(busy_cpu) && !has_misfit))) + continue; + + /* if helping farthest cluster, kick a middle */ + if (num_sched_clusters > 2 && + ((wrq->cluster->id == 0 && src_cluster_id == num_sched_clusters - 1) || + (wrq->cluster->id == num_sched_clusters - 1 && src_cluster_id == 0))) { + first_idle = find_first_idle_if_others_are_busy(); if (first_idle != -1) { walt_kick_cpu(first_idle); - } else if (walt_rotation_enabled) { - goto found_busy_cpu; + } else { + if (walt_rotation_enabled && + capacity_orig_of(this_cpu) > + capacity_orig_of(busy_cpu)) { + /* + * When BTR active help + * smallest immediately + */ + goto found_busy_cpu; + } } + } else { + goto found_busy_cpu; } - } else if (order_index == 2) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - /* help gold only if prime has had enough idle or gold has a misfit */ - has_misfit = false; - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; - - /* help the farthest cluster indirectly if it needs help */ - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1) { - first_idle = - find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - if (first_idle != -1) { - walt_kick_cpu(first_idle); - } else if (walt_rotation_enabled) { - goto found_busy_cpu; - } - } - } else { - busy_cpu = - walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - if (enough_idle) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - } - - has_misfit = false; - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; } goto unlock; From a09a292a85354e1b30b9846a65d710c1daf2f3c8 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 12 Oct 2022 12:00:01 -0700 Subject: [PATCH 10/10] sched/walt: expand cpu arrays Build_cpu_array function is rewritten to setup the cpu arrays manually rather than automatically based on cluster id and capacity. This allows for more flexible setup of cpu arrays. While at it also start supporting four cluster systems. Change-Id: I3932d9bbd0de95d6356ee298e733e7c5e799974f Signed-off-by: Shaleen Agrawal Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 77 ++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2788c9323459..8fd8b5ad2138 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2582,16 +2582,39 @@ static void update_all_clusters_stats(void) static bool walt_clusters_parsed; cpumask_t __read_mostly **cpu_array; +u8 cpu_arrays_init_x11[1][1] = { + {0}, /* S */ +}; + +u8 cpu_arrays_init_x22[2][2] = { + {0, 1}, /* S G */ + {1, 0}, /* G S */ +}; + +u8 cpu_arrays_init_x33[3][3] = { + {0, 1, 2}, /* S G P */ + {1, 2, 0}, /* G P S */ + {2, 1, 0}, /* P G S */ +}; + +u8 cpu_arrays_init_x44[4][4] = { + {0, 2, 1, 3}, /* S T G P */ + {1, 2, 3, 0}, /* G T P S */ + {2, 3, 1, 0}, /* T P G S */ + {3, 1, 2, 0}, /* P G T S */ +}; + static void init_cpu_array(void) { int i; + int rows = num_sched_clusters; - cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *), + cpu_array = kcalloc(rows, sizeof(cpumask_t *), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array) WALT_PANIC(1); - for (i = 0; i < num_sched_clusters; i++) { + for (i = 0; i < rows; i++) { cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array[i]) @@ -2601,35 +2624,35 @@ static void init_cpu_array(void) static void build_cpu_array(void) { - int i; + u8 *select_init_list; + u8 id; + int i, j; if (!cpu_array) WALT_PANIC(1); - /* Construct cpu_array row by row */ + + switch (num_sched_clusters) { + case 1: + select_init_list = (u8 *)cpu_arrays_init_x11; + break; + case 2: + select_init_list = (u8 *)cpu_arrays_init_x22; + break; + case 3: + select_init_list = (u8 *)cpu_arrays_init_x33; + break; + case 4: + select_init_list = (u8 *)cpu_arrays_init_x44; + break; + default: + pr_err("unsupported num clusters=%d\n", num_sched_clusters); + WALT_PANIC(1); + } + for (i = 0; i < num_sched_clusters; i++) { - int j, k = 1; - - /* Fill out first column with appropriate cpu arrays */ - cpumask_copy(&cpu_array[i][0], &sched_cluster[i]->cpus); - /* - * k starts from column 1 because 0 is filled - * Fill clusters for the rest of the row, - * above i in ascending order - */ - for (j = i + 1; j < num_sched_clusters; j++) { - cpumask_copy(&cpu_array[i][k], - &sched_cluster[j]->cpus); - k++; - } - - /* - * k starts from where we left off above. - * Fill clusters below i in descending order. - */ - for (j = i - 1; j >= 0; j--) { - cpumask_copy(&cpu_array[i][k], - &sched_cluster[j]->cpus); - k++; + for (j = 0; j < num_sched_clusters; j++) { + id = select_init_list[i * num_sched_clusters + j]; + cpumask_copy(&cpu_array[i][j], &sched_cluster[id]->cpus); } } }