diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 8a213e937820..56d0ea018597 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -20,7 +20,7 @@ enum pause_reason { #if IS_ENABLED(CONFIG_SCHED_WALT) #define MAX_CPUS_PER_CLUSTER 6 -#define MAX_CLUSTERS 3 +#define MAX_CLUSTERS 4 struct core_ctl_notif_data { unsigned int nr_big; diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index a7dee202065e..21726f06ba9f 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "walt.h" #include "trace.h" @@ -37,8 +38,8 @@ struct cluster_data { unsigned int need_cpus; unsigned int task_thres; unsigned int max_nr; - unsigned int nr_prev_assist; - unsigned int nr_prev_assist_thresh; + unsigned int nr_assist; + unsigned int nr_assist_thresh; s64 need_ts; struct list_head lru; bool enable; @@ -47,6 +48,10 @@ struct cluster_data { unsigned int boost; struct kobject kobj; unsigned int strict_nrrun; + cpumask_t nrrun_cpu_mask; + cpumask_t nrrun_cpu_misfit_mask; + cpumask_t assist_cpu_mask; + cpumask_t assist_cpu_misfit_mask; }; struct cpu_data { @@ -80,11 +85,14 @@ static DEFINE_SPINLOCK(state_lock); static void apply_need(struct cluster_data *state); static void wake_up_core_ctl_thread(void); static bool initialized; +static bool assist_params_initialized; ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); static unsigned int last_nr_big; static unsigned int get_active_cpu_count(const struct cluster_data *cluster); +static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster); +static unsigned int get_active_count(cpumask_t *cpumask); static void __ref do_core_ctl(void); /* ========================= sysfs interface =========================== */ @@ -163,13 +171,13 @@ static ssize_t store_task_thres(struct cluster_data *state, return count; } -static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state, +static ssize_t show_nr_assist_thresh(const struct cluster_data *state, char *buf) { - return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh); + return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_assist_thresh); } -static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, +static ssize_t store_nr_assist_thresh(struct cluster_data *state, const char *buf, size_t count) { unsigned int val; @@ -177,7 +185,7 @@ static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, if (sscanf(buf, "%u\n", &val) != 1) return -EINVAL; - state->nr_prev_assist_thresh = val; + state->nr_assist_thresh = val; apply_need(state); return count; @@ -399,6 +407,129 @@ static ssize_t show_not_preferred(const struct cluster_data *state, char *buf) return count; } +DECLARE_BITMAP(temp_bitmap, WALT_NR_CPUS); + +static ssize_t store_nrrun_cpu_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask = 0xFF; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->nrrun_cpu_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_nrrun_cpu_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_mask)))); + + return ret; +} + +static ssize_t store_nrrun_cpu_misfit_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->nrrun_cpu_misfit_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_nrrun_cpu_misfit_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->nrrun_cpu_misfit_mask)))); + + return ret; +} + +static ssize_t store_assist_cpu_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->assist_cpu_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_assist_cpu_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", (*(cpumask_bits(&state->assist_cpu_mask)))); + + return ret; +} + +static ssize_t store_assist_cpu_misfit_mask(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned long bitmask; + const unsigned long *bitmaskp = &bitmask; + unsigned long flags; + int ret = 0; + + ret = kstrtoul(buf, 0, (unsigned long *)bitmaskp); + if (ret < 0) + return ret; + + bitmap_copy(temp_bitmap, bitmaskp, 8); + + spin_lock_irqsave(&state_lock, flags); + cpumask_copy(&state->assist_cpu_misfit_mask, to_cpumask(temp_bitmap)); + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_assist_cpu_misfit_mask(const struct cluster_data *state, char *buf) +{ + int ret = 0; + + ret = scnprintf(buf, PAGE_SIZE, "0x%x\n", + (*(cpumask_bits(&state->assist_cpu_misfit_mask)))); + + return ret; +} + struct core_ctl_attr { struct attribute attr; ssize_t (*show)(const struct cluster_data *cd, char *c); @@ -420,12 +551,16 @@ core_ctl_attr_rw(offline_delay_ms); core_ctl_attr_rw(busy_up_thres); core_ctl_attr_rw(busy_down_thres); core_ctl_attr_rw(task_thres); -core_ctl_attr_rw(nr_prev_assist_thresh); +core_ctl_attr_rw(nr_assist_thresh); core_ctl_attr_ro(need_cpus); core_ctl_attr_ro(active_cpus); core_ctl_attr_ro(global_state); core_ctl_attr_rw(not_preferred); core_ctl_attr_rw(enable); +core_ctl_attr_rw(nrrun_cpu_mask); +core_ctl_attr_rw(nrrun_cpu_misfit_mask); +core_ctl_attr_rw(assist_cpu_mask); +core_ctl_attr_rw(assist_cpu_misfit_mask); static struct attribute *default_attrs[] = { &min_cpus.attr, @@ -434,12 +569,16 @@ static struct attribute *default_attrs[] = { &busy_up_thres.attr, &busy_down_thres.attr, &task_thres.attr, - &nr_prev_assist_thresh.attr, + &nr_assist_thresh.attr, &enable.attr, &need_cpus.attr, &active_cpus.attr, &global_state.attr, ¬_preferred.attr, + &nrrun_cpu_mask.attr, + &nrrun_cpu_misfit_mask.attr, + &assist_cpu_mask.attr, + &assist_cpu_misfit_mask.attr, NULL }; ATTRIBUTE_GROUPS(default); @@ -485,11 +624,16 @@ static struct kobj_type ktype_core_ctl = { static struct sched_avg_stats *nr_stats; -/* - * nr_need: +/** + * compute_cluster_nr_run: + * @index: cluster index + * * Number of tasks running on this cluster plus - * tasks running on higher capacity clusters. - * To find out CPUs needed from this cluster. + * tasks running on monitored clusters to find + * out CPUs needed from this cluster. Typically + * the other cpus that are monitored are from + * higher capacity clusters, and full clusters + * are considered. * * For example: * On dual cluster system with 4 min capacity @@ -504,56 +648,51 @@ static struct sched_avg_stats *nr_stats; * can be ready to accommodate tasks running on max * capacity CPUs if the demand of tasks goes down. */ -static int compute_cluster_nr_need(int index) +static int compute_cluster_nr_run(int index) { int cpu; struct cluster_data *cluster; int nr_need = 0; - for_each_cluster(cluster, index) { - for_each_cpu(cpu, &cluster->cpu_mask) - nr_need += nr_stats[cpu].nr; - } + cluster = &cluster_state[index]; + + for_each_cpu(cpu, &cluster->nrrun_cpu_mask) + nr_need += nr_stats[cpu].nr; return nr_need; } -/* - * prev_misfit_need: - * Tasks running on smaller capacity cluster which - * needs to be migrated to higher capacity cluster. - * To find out how many tasks need higher capacity CPUs. - * - * For example: - * On dual cluster system with 4 min capacity - * CPUs and 4 max capacity CPUs, if there are - * 2 small tasks and 2 big tasks running on - * min capacity CPUs and no tasks running on - * max cpacity, prev_misfit_need of min capacity - * cluster will be 0 and prev_misfit_need of - * max capacity cluster will be 2. +/** + * compute_cluster_nr_misfit: + * @index: cluster index + * Tasks running on cpus which this cluster is monitoring, + * that need to be migrated to this cluster. Typically, + * lower capacity cpus are monitored, and full clusters + * are considered. */ -static int compute_prev_cluster_misfit_need(int index) +static int compute_cluster_nr_misfit(int index) { int cpu; - struct cluster_data *prev_cluster; + struct cluster_data *cluster; int prev_misfit_need = 0; - /* - * Lowest capacity cluster does not have to - * accommodate any misfit tasks. - */ - if (index == 0) - return 0; + cluster = &cluster_state[index]; - prev_cluster = &cluster_state[index - 1]; - - for_each_cpu(cpu, &prev_cluster->cpu_mask) + for_each_cpu(cpu, &cluster->nrrun_cpu_misfit_mask) prev_misfit_need += nr_stats[cpu].nr_misfit; return prev_misfit_need; } +/** + * compute_cluster_max_nr + * @index: cluster index + * + * For each cpu in this cluster, determine the maximum + * number of tasks running on the cpu, and return the + * maximum number of tasks seen on a single cpu, for + * this cluster. + */ static int compute_cluster_max_nr(int index) { int cpu; @@ -566,6 +705,105 @@ static int compute_cluster_max_nr(int index) return max_nr; } +/** + * cluster->nr_assist (aka prev_nr_need_assist) = + * compute_cluster_nr_run_assist() + + * compute_cluster_nr_run_misfit_assist() - + * get_assist_active_cpu_count() + * + * nr_assist is the number of tasks that are eligible to run on + * the monitored cpus, but cannot run because of insufficient + * CPUs there. The cpus being monitored yielding this information + * used by compute_cluster_nr_run_assist() and + * compute_cluster_nr_run_misfit_assist() are assist_cpu_mask and + * assist_cpu_misfit_mask, respectively. + * + * cluster->nr_assist is zero if there are no paused cpus in this cluster. + * + * For example: + * + * If max capacity cluster masks are defined as + * assist_cpu_mask=0x70 and assist_cpu_misfit_mask=0x0F: + * + * On tri-cluster system with 4 min capacity CPUs, 3 intermediate + * capacity CPUs and 1 max capacity CPU, if there are 4 small + * tasks running on min capacity CPUs, 4 big tasks running on + * intermediate capacity CPUs and no tasks running on max capacity + * CPU, nr_run_misfit_assist for min & max capacity clusters will be + * 0, but for intermediate capacity cluster nr_run_assist will be 1 + * as it has 3 CPUs, but, there are 4 big tasks to be served. + * + * Since the max capacity cluster is monitoring intermediate + * clusters for number of tasks, and min capacity clusters + * for number of misfits and there are no misfits on min + * capacity cpus, this component is 0. Since there are 4 big + * tasks on intertmediate cap cpus but only 3 CPUs, nr_run_assist + * for the max capacity cluster is 1. + * + * As a further example, if the cluster had one misfit task in addition + * the small tasks on silvers, prime would have counted 1 task + * as a result of nr_run_misfit_assist CPUs, and 1 task as a result of + * nr_run_assist CPUs. + * + * In both cases the max capacity cpu is currently unpaused, nr_assist + * will be 0. + */ + +/** + * compute_cluster_nr_run_assist: + * @index: cluster index + * Tasks running on cpus that this cluster + * is assisting. Typically the cpus being + * monitored are lower capacity cpus, not + * including the current cluster. + */ +static int compute_cluster_nr_run_assist(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int nr_assist = 0; + + for_each_cpu(cpu, &cluster->assist_cpu_mask) + nr_assist += nr_stats[cpu].nr; + + return nr_assist; +} + +/** + * compute_cluster_nr_run_assist: + * @index: cluster index + * Tasks running on cpus that this cluster + * is assisting for misfits. Typically the + * cpus being monitored are lower capacity + * cpus, not including the current cluster. + * + * In a 3 cluster system, this means that prime + * would be monitoring golds for assistance with + * misfits. + */ +static int compute_cluster_nr_misfit_assist(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int nr_misfit_assist = 0; + + for_each_cpu(cpu, &cluster->assist_cpu_misfit_mask) + nr_misfit_assist += nr_stats[cpu].nr; + + return nr_misfit_assist; +} + + +/** + * cluster_real_big_tasks + * @index: cluster index + * + * Return the number of misfits on the lowest capacity + * cluster, or the number of tasks running on a bigger + * capacity cluster. This means that any task running + * on a non-min-capacity-cluster is considered a big + * task. + */ static int cluster_real_big_tasks(int index) { int nr_big = 0; @@ -583,60 +821,6 @@ static int cluster_real_big_tasks(int index) return nr_big; } -/* - * prev_nr_need_assist: - * Tasks that are eligible to run on the previous - * cluster but cannot run because of insufficient - * CPUs there. prev_nr_need_assist is indicative - * of number of CPUs in this cluster that should - * assist its previous cluster to makeup for - * insufficient CPUs there. - * - * For example: - * On tri-cluster system with 4 min capacity - * CPUs, 3 intermediate capacity CPUs and 1 - * max capacity CPU, if there are 4 small - * tasks running on min capacity CPUs, 4 big - * tasks running on intermediate capacity CPUs - * and no tasks running on max capacity CPU, - * prev_nr_need_assist for min & max capacity - * clusters will be 0, but, for intermediate - * capacity cluster prev_nr_need_assist will - * be 1 as it has 3 CPUs, but, there are 4 big - * tasks to be served. - */ -static int prev_cluster_nr_need_assist(int index) -{ - int need = 0; - int cpu; - struct cluster_data *prev_cluster; - - if (index == 0) - return 0; - - index--; - prev_cluster = &cluster_state[index]; - - /* - * Next cluster should not assist, while there are paused cpus - * in this cluster. - */ - if (cluster_paused_cpus(prev_cluster)) - return 0; - - for_each_cpu(cpu, &prev_cluster->cpu_mask) - need += nr_stats[cpu].nr; - - need += compute_prev_cluster_misfit_need(index); - - if (need > prev_cluster->active_cpus) - need = need - prev_cluster->active_cpus; - else - need = 0; - - return need; -} - /* * This is only implemented for min capacity cluster. * @@ -649,37 +833,39 @@ static int prev_cluster_nr_need_assist(int index) * capacity CPUs from the nr and consider the remaining nr as * strict and consider that many little CPUs are needed. */ -static int compute_cluster_nr_strict_need(int index) +static int compute_cluster_strict_nr_run(int index) { int cpu; struct cluster_data *cluster; int nr_strict_need = 0; + int nr_scaled = 0; if (index != 0) return 0; - for_each_cluster(cluster, index) { - int nr_scaled = 0; - int active_cpus = cluster->active_cpus; + cluster = &cluster_state[index]; - for_each_cpu(cpu, &cluster->cpu_mask) - nr_scaled += nr_stats[cpu].nr_scaled; + for_each_cpu(cpu, &cluster->nrrun_cpu_mask) + nr_scaled += nr_stats[cpu].nr_scaled; - nr_scaled /= 100; + nr_scaled /= 100; - /* - * For little cluster, nr_scaled becomes the nr_strict, - * for other cluster, overflow is counted towards - * the little cluster need. - */ - if (index == 0) - nr_strict_need += nr_scaled; - else - nr_strict_need += max(0, nr_scaled - active_cpus); + /* + * For little cluster, nr_scaled becomes the nr_strict, + * for other cluster, overflow is counted towards + * the little cluster need. + */ + if (index == 0) { + nr_strict_need += nr_scaled; + } else { + int active_cpus = get_active_count(&cluster->nrrun_cpu_mask); + + nr_strict_need += max(0, nr_scaled - active_cpus); } return nr_strict_need; } + static void update_running_avg(void) { struct cluster_data *cluster; @@ -691,24 +877,35 @@ static void update_running_avg(void) spin_lock_irqsave(&state_lock, flags); for_each_cluster(cluster, index) { - int nr_need, prev_misfit_need; + int nr_need, nr_misfit_need; + int nr_assist_need, nr_misfit_assist_need, nr_assist_active; if (!cluster->inited) continue; - nr_need = compute_cluster_nr_need(index); - prev_misfit_need = compute_prev_cluster_misfit_need(index); + nr_need = compute_cluster_nr_run(index); + nr_misfit_need = compute_cluster_nr_misfit(index); - cluster->nrrun = nr_need + prev_misfit_need; + cluster->nrrun = nr_need + nr_misfit_need; cluster->max_nr = compute_cluster_max_nr(index); - cluster->nr_prev_assist = prev_cluster_nr_need_assist(index); - cluster->strict_nrrun = compute_cluster_nr_strict_need(index); + nr_assist_need = compute_cluster_nr_run_assist(index); + nr_misfit_assist_need = compute_cluster_nr_misfit_assist(index); + + cluster->strict_nrrun = compute_cluster_strict_nr_run(index); + nr_assist_active = get_assist_active_cpu_count(cluster); + + if (!cpumask_intersects(&cluster->assist_cpu_mask, &cpus_paused_by_us) && + nr_assist_need + nr_misfit_assist_need > nr_assist_active) + cluster->nr_assist = nr_assist_need + + nr_misfit_assist_need - nr_assist_active; + else + cluster->nr_assist = 0; trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need, - prev_misfit_need, + nr_misfit_need, cluster->nrrun, cluster->max_nr, - cluster->nr_prev_assist); + cluster->nr_assist); big_avg += cluster_real_big_tasks(index); } @@ -731,8 +928,8 @@ static unsigned int apply_task_need(const struct cluster_data *cluster, * resume as many cores as the previous cluster * needs assistance with. */ - if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh) - new_need = new_need + cluster->nr_prev_assist; + if (cluster->nr_assist >= cluster->nr_assist_thresh) + new_need = new_need + cluster->nr_assist; /* only resume more cores if there are tasks to run */ if (cluster->nrrun > new_need) @@ -765,6 +962,14 @@ static unsigned int apply_limits(const struct cluster_data *cluster, return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus); } +static unsigned int get_active_count(cpumask_t *cpumask) +{ + cpumask_t cpus; + + cpumask_andnot(&cpus, cpumask, cpu_halt_mask); + return cpumask_weight(&cpus); +} + static unsigned int get_active_cpu_count(const struct cluster_data *cluster) { cpumask_t cpus; @@ -773,6 +978,14 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster) return cpumask_weight(&cpus); } +static unsigned int get_assist_active_cpu_count(const struct cluster_data *cluster) +{ + cpumask_t cpus; + + cpumask_andnot(&cpus, &cluster->assist_cpu_mask, cpu_halt_mask); + return cpumask_weight(&cpus); +} + static bool is_active(const struct cpu_data *state) { return cpu_active(state->cpu) && !cpu_halted(state->cpu); @@ -955,6 +1168,31 @@ static void core_ctl_call_notifier(void) atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata); } +/** + * core_ctl_check_masks_set + * + * return true if all clusters have updated values for their appropriate + * masks. + */ +static bool core_ctl_check_masks_set(void) +{ + int index = 0; + struct cluster_data *cluster; + int possible_cpus = cpumask_weight(cpu_possible_mask); + int all_masks_set = true; + + for_each_cluster(cluster, index) { + if (cpumask_weight(&cluster->nrrun_cpu_mask) > possible_cpus || + cpumask_weight(&cluster->nrrun_cpu_misfit_mask) > possible_cpus || + cpumask_weight(&cluster->assist_cpu_mask) > possible_cpus || + cpumask_weight(&cluster->assist_cpu_misfit_mask) > possible_cpus) { + all_masks_set = false; + break; + } + } + + return all_masks_set; +} /* * sched_get_nr_running_avg will wipe out previous statistics and * update it to the values computed since the last call. @@ -975,6 +1213,11 @@ void core_ctl_check(u64 window_start) if (unlikely(!initialized)) return; + if (unlikely(!assist_params_initialized)) { + assist_params_initialized = core_ctl_check_masks_set(); + return; + } + if (window_start == core_ctl_check_timestamp) return; @@ -1306,11 +1549,22 @@ static int cluster_init(const struct cpumask *mask) cluster->need_cpus = cluster->num_cpus; cluster->offline_delay_ms = 100; cluster->task_thres = UINT_MAX; - cluster->nr_prev_assist_thresh = UINT_MAX; + cluster->nr_assist_thresh = UINT_MAX; cluster->nrrun = cluster->num_cpus; - cluster->enable = true; + cluster->enable = false; cluster->nr_not_preferred_cpus = 0; cluster->strict_nrrun = 0; + + /* + * set all cpus in the cluster. this is an invalid state + * and core control will not be considered initialized until + * this state is no longer true (all masks must be written). + */ + cpumask_setall(&cluster->nrrun_cpu_mask); + cpumask_setall(&cluster->nrrun_cpu_misfit_mask); + cpumask_setall(&cluster->assist_cpu_mask); + cpumask_setall(&cluster->assist_cpu_misfit_mask); + INIT_LIST_HEAD(&cluster->lru); for_each_cpu(cpu, mask) { diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 38b0c731eae4..4762798ab7ee 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -365,39 +365,25 @@ static void sched_update_updown_migrate_values(bool up) { int i = 0, cpu; struct walt_sched_cluster *cluster; - int cap_margin_levels = num_sched_clusters - 1; - if (cap_margin_levels > 1) { + for_each_sched_cluster(cluster) { /* * No need to worry about CPUs in last cluster * if there are more than 2 clusters in the system */ - for_each_sched_cluster(cluster) { - for_each_cpu(cpu, &cluster->cpus) { - if (up) - sched_capacity_margin_up[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_up_pct[i]; - else - sched_capacity_margin_down[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_dn_pct[i]; - } - - if (++i >= cap_margin_levels) - break; - } - } else { - for_each_possible_cpu(cpu) { + for_each_cpu(cpu, &cluster->cpus) { if (up) sched_capacity_margin_up[cpu] = - SCHED_FIXEDPOINT_SCALE * 100 / - sysctl_sched_capacity_margin_up_pct[0]; + sysctl_sched_capacity_margin_up_pct[i]; else sched_capacity_margin_down[cpu] = - sysctl_sched_capacity_margin_dn_pct[0]; + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_dn_pct[i]; } + + if (++i >= num_sched_clusters - 1) + break; } } @@ -468,6 +454,28 @@ int sched_updown_migrate_handler(struct ctl_table *table, int write, return ret; } +static void sched_update_updown_early_migrate_values(bool up) +{ + int i = 0, cpu; + struct walt_sched_cluster *cluster; + + for_each_sched_cluster(cluster) { + /* + * No need to worry about CPUs in last cluster + * if there are more than 2 clusters in the system + */ + for_each_cpu(cpu, &cluster->cpus) { + if (up) + sched_capacity_margin_early_up[cpu] = sysctl_sched_early_up[i]; + else + sched_capacity_margin_early_down[cpu] = sysctl_sched_early_down[i]; + } + + if (++i >= num_sched_clusters - 1) + break; + } +} + int sched_updown_early_migrate_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) @@ -504,7 +512,7 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write, } } - /* check up pct is greater than dn pct */ + /* check up thresh is greater than dn thresh */ if (data == &sysctl_sched_early_up[0]) { for (i = 0; i < cap_margin_levels; i++) { if (val[i] >= sysctl_sched_early_down[i]) { @@ -525,6 +533,8 @@ int sched_updown_early_migrate_handler(struct ctl_table *table, int write, for (i = 0; i < cap_margin_levels; i++) data[i] = val[i]; + /* update individual cpu thresholds */ + sched_update_updown_early_migrate_values(data == &sysctl_sched_early_up[0]); unlock_mutex: mutex_unlock(&mutex); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4507cb2d02a2..c3f1d26a7274 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -44,8 +44,6 @@ const char *migrate_type_names[] = { #define EARLY_DETECTION_DURATION 9500000 #define MAX_NUM_CGROUP_COLOC_ID 20 -#define MAX_NR_CLUSTERS 3 - #define NEW_TASK_ACTIVE_TIME 100000000 DEFINE_PER_CPU(struct walt_rq, walt_rq); @@ -2534,7 +2532,7 @@ static inline void assign_cluster_ids(struct list_head *head) sched_cluster[pos++] = cluster; } - WARN_ON(pos > MAX_NR_CLUSTERS); + WARN_ON(pos > MAX_CLUSTERS); } static inline void @@ -2585,16 +2583,39 @@ static void update_all_clusters_stats(void) static bool walt_clusters_parsed; cpumask_t __read_mostly **cpu_array; +u8 cpu_arrays_init_x11[1][1] = { + {0}, /* S */ +}; + +u8 cpu_arrays_init_x22[2][2] = { + {0, 1}, /* S G */ + {1, 0}, /* G S */ +}; + +u8 cpu_arrays_init_x33[3][3] = { + {0, 1, 2}, /* S G P */ + {1, 2, 0}, /* G P S */ + {2, 1, 0}, /* P G S */ +}; + +u8 cpu_arrays_init_x44[4][4] = { + {0, 2, 1, 3}, /* S T G P */ + {1, 2, 3, 0}, /* G T P S */ + {2, 3, 1, 0}, /* T P G S */ + {3, 1, 2, 0}, /* P G T S */ +}; + static void init_cpu_array(void) { int i; + int rows = num_sched_clusters; - cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *), + cpu_array = kcalloc(rows, sizeof(cpumask_t *), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array) WALT_PANIC(1); - for (i = 0; i < num_sched_clusters; i++) { + for (i = 0; i < rows; i++) { cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array[i]) @@ -2604,35 +2625,35 @@ static void init_cpu_array(void) static void build_cpu_array(void) { - int i; + u8 *select_init_list; + u8 id; + int i, j; if (!cpu_array) WALT_PANIC(1); - /* Construct cpu_array row by row */ + + switch (num_sched_clusters) { + case 1: + select_init_list = (u8 *)cpu_arrays_init_x11; + break; + case 2: + select_init_list = (u8 *)cpu_arrays_init_x22; + break; + case 3: + select_init_list = (u8 *)cpu_arrays_init_x33; + break; + case 4: + select_init_list = (u8 *)cpu_arrays_init_x44; + break; + default: + pr_err("unsupported num clusters=%d\n", num_sched_clusters); + WALT_PANIC(1); + } + for (i = 0; i < num_sched_clusters; i++) { - int j, k = 1; - - /* Fill out first column with appropriate cpu arrays */ - cpumask_copy(&cpu_array[i][0], &sched_cluster[i]->cpus); - /* - * k starts from column 1 because 0 is filled - * Fill clusters for the rest of the row, - * above i in ascending order - */ - for (j = i + 1; j < num_sched_clusters; j++) { - cpumask_copy(&cpu_array[i][k], - &sched_cluster[j]->cpus); - k++; - } - - /* - * k starts from where we left off above. - * Fill clusters below i in descending order. - */ - for (j = i - 1; j >= 0; j--) { - cpumask_copy(&cpu_array[i][k], - &sched_cluster[j]->cpus); - k++; + for (j = 0; j < num_sched_clusters; j++) { + id = select_init_list[i * num_sched_clusters + j]; + cpumask_copy(&cpu_array[i][j], &sched_cluster[id]->cpus); } } } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 6274f7a49071..5e65413c9981 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -770,51 +770,45 @@ static bool check_for_higher_capacity(int cpu1, int cpu2) return capacity_orig_of(cpu1) > capacity_orig_of(cpu2); } -extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; -extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; +/* Migration margins for topapp */ +extern unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS]; +extern unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS]; static inline bool task_fits_capacity(struct task_struct *p, - long capacity, - int cpu) + int dst_cpu) { unsigned int margin; + unsigned long capacity = capacity_orig_of(dst_cpu); /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. */ - if (check_for_higher_capacity(task_cpu(p), cpu)) { - margin = sched_capacity_margin_down[cpu]; + if (check_for_higher_capacity(task_cpu(p), dst_cpu)) { + margin = sched_capacity_margin_down[dst_cpu]; if (task_in_related_thread_group(p)) { - if (is_min_cluster_cpu(cpu)) - margin = sysctl_sched_early_down[0]; - else if (!is_max_cluster_cpu(cpu)) - margin = sysctl_sched_early_down[1]; + margin = sched_capacity_margin_early_down[dst_cpu]; } } else { margin = sched_capacity_margin_up[task_cpu(p)]; if (task_in_related_thread_group(p)) { - if (is_min_cluster_cpu(task_cpu(p))) - margin = sysctl_sched_early_up[0]; - else if (!is_max_cluster_cpu(task_cpu(p))) - margin = sysctl_sched_early_up[1]; + margin = sched_capacity_margin_early_up[task_cpu(p)]; } } return capacity * 1024 > uclamp_task_util(p) * margin; } -static inline bool task_fits_max(struct task_struct *p, int cpu) +static inline bool task_fits_max(struct task_struct *p, int dst_cpu) { - unsigned long capacity = capacity_orig_of(cpu); unsigned long task_boost = per_task_boost(p); - if (is_max_cluster_cpu(cpu)) + if (is_max_cluster_cpu(dst_cpu)) return true; - if (is_min_cluster_cpu(cpu)) { + if (is_min_cluster_cpu(dst_cpu)) { if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || task_boost > 0 || walt_uclamp_boosted(p) || - walt_should_kick_upmigrate(p, cpu)) + walt_should_kick_upmigrate(p, dst_cpu)) return false; } else { /* mid cap cpu */ if (task_boost > TASK_BOOST_ON_MID) @@ -824,7 +818,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) return true; } - return task_fits_capacity(p, capacity, cpu); + return task_fits_capacity(p, dst_cpu); } extern struct sched_avg_stats *sched_get_nr_running_avg(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 456f5560743b..e88ba99b7bde 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -61,6 +61,14 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ }; +/* Migration margins for topapp */ +unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */ +}; +unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ +}; + static inline bool bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { @@ -72,19 +80,17 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) return base_test && start_cap_test; } -static inline bool task_demand_fits(struct task_struct *p, int cpu) +static inline bool task_demand_fits(struct task_struct *p, int dst_cpu) { - unsigned long capacity = capacity_orig_of(cpu); - - if (is_max_cluster_cpu(cpu)) + if (is_max_cluster_cpu(dst_cpu)) return true; if (!task_in_related_thread_group(p) && p->prio >= 124 && - !is_min_cluster_cpu(cpu) && !is_max_cluster_cpu(cpu)) { + !is_min_cluster_cpu(dst_cpu) && !is_max_cluster_cpu(dst_cpu)) { /* a non topapp low prio task fits on gold */ return true; } - return task_fits_capacity(p, capacity, cpu); + return task_fits_capacity(p, dst_cpu); } struct find_best_target_env { @@ -165,8 +171,6 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, int *end_index, int per_task_boost, bool is_uclamp_boosted, bool *energy_eval_needed) { - int i = 0; - *order_index = 0; *end_index = 0; @@ -182,9 +186,12 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, if (is_full_throttle_boost()) { *energy_eval_needed = false; *order_index = num_sched_clusters - 1; - if ((*order_index > 1) && task_demand_fits(p, - cpumask_first(&cpu_array[*order_index][1]))) - *end_index = 1; + *end_index = num_sched_clusters - 2; + + for (; *end_index >= 0; (*end_index)--) + if (task_demand_fits(p, + cpumask_first(&cpu_array[*order_index][*end_index]))) + break; return; } @@ -193,19 +200,19 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, walt_task_skip_min_cpu(p)) { *energy_eval_needed = false; *order_index = 1; + *end_index = max(0, num_sched_clusters - 3); + if (sysctl_sched_asymcap_boost) { - *end_index = 1; + (*end_index)++; return; } } - for (i = *order_index ; i < num_sched_clusters - 1; i++) { - if (task_demand_fits(p, cpumask_first(&cpu_array[i][0]))) + for (; *order_index < num_sched_clusters - 1; (*order_index)++) { + if (task_demand_fits(p, cpumask_first(&cpu_array[*order_index][0]))) break; } - *order_index = i; - if (*order_index == 0 && (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && !(p->in_iowait && task_in_related_thread_group(p)) && diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index be8027a21d54..8b031c87a4f8 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -422,19 +422,34 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) /* * find_first_idle_if_others_are_busy * - * Get an idle cpu in the src_mask, iif the other cpus are busy - * with larger tasks or more than one task. + * Get an idle cpu in the middle clusters * * Returns -1 if there are no idle cpus in the mask, or if there * is a CPU that is about to be come newly idle. Returns * if there is an idle cpu in a cluster that's not about to do * newly idle load balancing. + * Also returns -1 if called on a single or dual cluster system */ -static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) +static int find_first_idle_if_others_are_busy(void) { int i, first_idle = -1; + struct cpumask src_mask; + + cpumask_clear(&src_mask); + for (i = 0; i < num_sched_clusters; i++) { + if (i == 0 || i == num_sched_clusters - 1) + continue; + else + cpumask_or(&src_mask, &src_mask, &cpu_array[0][i]); + } + + for_each_cpu(i, &src_mask) { + if (!cpu_active(i)) + continue; + + if (cpu_halted(i)) + continue; - for_each_cpu(i, src_mask) { if (available_idle_cpu(i)) first_idle = i; @@ -800,9 +815,10 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int order_index; int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); - bool help_min_cap = false, find_next_cluster = false; + bool help_min_cap = false; int first_idle; int has_misfit = 0; + int i; if (unlikely(walt_disabled)) return; @@ -858,98 +874,48 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * can be queued remotely, so keep a check on nr_running * and bail out. */ - if (num_sched_clusters <= 2) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - if (num_sched_clusters == 2) { - has_misfit = false; - find_next_cluster = (order_index == 0) ? enough_idle : 1; - if (find_next_cluster) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, - &cpu_array[order_index][1], &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; - } - } + order_index = wrq->cluster->id; + for (i = 0; i < num_sched_clusters; i++) { + int first_cpu = cpumask_first(&cpu_array[order_index][i]); + struct walt_rq *src_wrq = &per_cpu(walt_rq, first_cpu); + int src_cluster_id = src_wrq->cluster->id; - goto unlock; - } - - if (order_index == 0) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - if (enough_idle) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - } - - /* - * help the farthest cluster by kicking an idle cpu in the next - * cluster. In case no idle is found, pull it in. + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][i], + &has_misfit, true); + if (busy_cpu == -1) + continue; + /* when not enough idle + * Small should not help big. + * Big should help small ONLY is mifit is present. + * Same capacity cpus should help each other */ - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1) { - first_idle = - find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + if (!enough_idle && + (capacity_orig_of(this_cpu) < capacity_orig_of(busy_cpu) || + (capacity_orig_of(this_cpu) > capacity_orig_of(busy_cpu) && !has_misfit))) + continue; + + /* if helping farthest cluster, kick a middle */ + if (num_sched_clusters > 2 && + ((wrq->cluster->id == 0 && src_cluster_id == num_sched_clusters - 1) || + (wrq->cluster->id == num_sched_clusters - 1 && src_cluster_id == 0))) { + first_idle = find_first_idle_if_others_are_busy(); if (first_idle != -1) { walt_kick_cpu(first_idle); - } else if (walt_rotation_enabled) { - goto found_busy_cpu; + } else { + if (walt_rotation_enabled && + capacity_orig_of(this_cpu) > + capacity_orig_of(busy_cpu)) { + /* + * When BTR active help + * smallest immediately + */ + goto found_busy_cpu; + } } + } else { + goto found_busy_cpu; } - } else if (order_index == 2) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - /* help gold only if prime has had enough idle or gold has a misfit */ - has_misfit = false; - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; - - /* help the farthest cluster indirectly if it needs help */ - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1) { - first_idle = - find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - if (first_idle != -1) { - walt_kick_cpu(first_idle); - } else if (walt_rotation_enabled) { - goto found_busy_cpu; - } - } - } else { - busy_cpu = - walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - - if (enough_idle) { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit, true); - if (busy_cpu != -1) - goto found_busy_cpu; - } - - has_misfit = false; - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit, true); - if (busy_cpu != -1 && (enough_idle || has_misfit)) - goto found_busy_cpu; } goto unlock;