diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 54b22401b581..3b5d7109ca15 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -188,6 +188,25 @@ static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws, wg_policy->last_ws = curr_ws; } +/* + * if waltgov is initialized, return the avg_cap seen + * over the last window. return 0 otherwise. + */ +unsigned int waltgov_get_avg_cap(unsigned int cpu) +{ + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + + /* if the policy is not initialized, or the callback + * is not initialized. callback is initialized on + * start of waltgov, erased on stop of waltgov. + */ + if (!wg_policy || !wg_cpu->cb.func) + return 0; + + return wg_policy->avg_cap; +} + static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, unsigned int next_freq) { diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index c18e5c2234a9..4e5a267d2b17 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -56,6 +56,16 @@ unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster) return cluster_util_pct; } +unsigned int sched_get_cpu_avg_cap(int cpu) +{ + unsigned int cpu_avg_cap = 0; + + if (cpu < WALT_NR_CPUS) + cpu_avg_cap = stats[cpu].avg_cap; + + return cpu_avg_cap; +} + /** * sched_get_nr_running_avg * @return: Average nr_running, iowait and nr_big_tasks value since last poll. @@ -109,6 +119,7 @@ struct sched_avg_stats *sched_get_nr_running_avg(void) NR_THRESHOLD_PCT), 100); stats[cpu].nr_max = per_cpu(nr_max, cpu); stats[cpu].nr_scaled = tmp_nr; + stats[cpu].avg_cap = waltgov_get_avg_cap(cpu); trace_sched_get_nr_running_avg(cpu, stats[cpu].nr, stats[cpu].nr_misfit, stats[cpu].nr_max, diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 1be0123ebcbe..310956da94e0 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -72,6 +72,8 @@ unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; unsigned int sysctl_sched_asymcap_boost; unsigned int sysctl_sched_long_running_rt_task_ms; +unsigned int sysctl_sched_idle_enough; +unsigned int sysctl_sched_cluster_util_thres_pct; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -893,6 +895,24 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_cluster_util_thres_pct", + .data = &sysctl_sched_cluster_util_thres_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_idle_enough", + .data = &sysctl_sched_idle_enough, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, { .procname = "sched_long_running_rt_task_ms", .data = &sysctl_sched_long_running_rt_task_ms, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 696ad1a99d68..bf159a4f3fb3 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -343,6 +343,98 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, static void rollover_cpu_window(struct rq *rq, bool full_window); static void rollover_top_tasks(struct rq *rq, bool full_window); +/* + * if last window's average capacity is less than or + * equal to the current capacity, return true. + */ +static inline bool is_cpufreq_avg_or_higher(int cpu) +{ + unsigned int avg_cap = sched_get_cpu_avg_cap(cpu); + unsigned int cur_cap = capacity_curr_of(cpu); + + if (cur_cap >= avg_cap && avg_cap != 0) + return true; + + return false; +} + +/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. + * @start_cpu: The cpu from the cluster to choose from + * + * If the cluster has a 32bit capable cpu return it regardless + * of whether it is halted or not. + * + * If the cluster does not have a 32 bit capable cpu, find the + * first unhalted, active cpu in this cluster. + */ +int walt_find_cluster_packing_cpu(int start_cpu) +{ + struct rq *rq = cpu_rq(start_cpu); + struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + cpumask_t unhalted_cpus; + cpumask_t cluster_32bit_cpus; + + /* find all 32 bit capable cpus in this cluster */ + cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); + + /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ + if (cpumask_weight(&cluster_32bit_cpus) > 0) + return cpumask_first(&cluster_32bit_cpus); + + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + + /* return the first found unhalted, active cpu, in this cluster */ + return cpumask_first(&unhalted_cpus); +} + +/* for cfs and rt, determine if packing_cpu should be used */ +bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) +{ + struct rq *rq; + struct walt_rq *wrq; + struct walt_sched_cluster *cluster; + + /* packing cpu must be a valid cpu for runqueue lookup */ + if (packing_cpu >= nr_cpu_ids) + return false; + + rq = cpu_rq(packing_cpu); + wrq = (struct walt_rq *)rq->android_vendor_data1; + cluster = wrq->cluster; + + /* if idle_enough feature is not enabled */ + if (!sysctl_sched_idle_enough) + return false; + + /* if cpu is not allowed for this task */ + if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) + return false; + + /* if cluster util is high */ + if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) + return false; + + /* if cpu utilization is high */ + if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) + return false; + + /* don't pack big tasks */ + if (task_util(p) >= sysctl_sched_idle_enough) + return false; + + /* if cpufreq is lower than the previous window */ + if (!is_cpufreq_avg_or_higher(packing_cpu)) + return false; + + /* the packing cpu can be used, so pack! */ + return true; +} + /* * Demand aggregation for frequency purpose: * diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 2c1487df0181..7c58e3d549b8 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -187,6 +187,9 @@ extern int max_possible_cluster_id; extern unsigned int __read_mostly sched_init_task_load_windows; extern unsigned int __read_mostly sched_load_granule; +extern unsigned int sysctl_sched_idle_enough; +extern unsigned int sysctl_sched_cluster_util_thres_pct; + /* 1ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_boost; extern unsigned int sysctl_sched_min_task_util_for_uclamp; @@ -345,6 +348,7 @@ struct sched_avg_stats { int nr_misfit; int nr_max; int nr_scaled; + u32 avg_cap; }; struct waltgov_callback { @@ -776,6 +780,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) extern struct sched_avg_stats *sched_get_nr_running_avg(void); extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster); +extern unsigned int sched_get_cpu_avg_cap(int cpu); +extern unsigned int waltgov_get_avg_cap(unsigned int cpu); extern void sched_update_hyst_times(void); extern void walt_rt_init(void); @@ -784,6 +790,8 @@ extern void walt_halt_init(void); extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint); +extern int walt_find_cluster_packing_cpu(int start_cpu); +extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p); static inline unsigned int cpu_max_possible_freq(int cpu) {