sched/walt: add functionality for choosing a packing cpu

Very small tasks can end up spread across all cpus
in a cluster, and cause excessive wfi and lpm exit
latencies. This can be reduced significantly by combining
these tasks on to a single cpu.

Create the functionality to be used in cfs/rt that can
make the decision as to whether a packing cpu should
be chosen. Add supporting sysctl nodes to tune the
thresholds for this.

Change-Id: I68723b8dc461efffeba3069bfd5d41724f071bd2
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>

kernel/sched/walt/cpufreq_walt.c

@@ -188,6 +188,25 @@ static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws,
 	wg_policy->last_ws = curr_ws;
 }
 
+/*
+ * if waltgov is initialized, return the avg_cap seen
+ * over the last window. return 0 otherwise.
+ */
+unsigned int waltgov_get_avg_cap(unsigned int cpu)
+{
+	struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu);
+	struct waltgov_policy *wg_policy = wg_cpu->wg_policy;
+
+	/* if the policy is not initialized, or the callback
+	 * is not initialized. callback is initialized on
+	 * start of waltgov, erased on stop of waltgov.
+	 */
+	if (!wg_policy || !wg_cpu->cb.func)
+		return 0;
+
+	return wg_policy->avg_cap;
+}
+
 static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time,
 			      unsigned int next_freq)
 {

kernel/sched/walt/sched_avg.c

@@ -56,6 +56,16 @@ unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster)
 	return cluster_util_pct;
 }
 
+unsigned int sched_get_cpu_avg_cap(int cpu)
+{
+	unsigned int cpu_avg_cap = 0;
+
+	if (cpu < WALT_NR_CPUS)
+		cpu_avg_cap = stats[cpu].avg_cap;
+
+	return cpu_avg_cap;
+}
+
 /**
  * sched_get_nr_running_avg
  * @return: Average nr_running, iowait and nr_big_tasks value since last poll.
@@ -109,6 +119,7 @@ struct sched_avg_stats *sched_get_nr_running_avg(void)
 						NR_THRESHOLD_PCT), 100);
 		stats[cpu].nr_max = per_cpu(nr_max, cpu);
 		stats[cpu].nr_scaled = tmp_nr;
+		stats[cpu].avg_cap = waltgov_get_avg_cap(cpu);
 
 		trace_sched_get_nr_running_avg(cpu, stats[cpu].nr,
 				stats[cpu].nr_misfit, stats[cpu].nr_max,

kernel/sched/walt/sysctl.c

@@ -72,6 +72,8 @@ unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1;
 unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000;
 unsigned int sysctl_sched_asymcap_boost;
 unsigned int sysctl_sched_long_running_rt_task_ms;
+unsigned int sysctl_sched_idle_enough;
+unsigned int sysctl_sched_cluster_util_thres_pct;
 
 /* range is [1 .. INT_MAX] */
 static int sysctl_task_read_pid = 1;
@@ -893,6 +895,24 @@ struct ctl_table walt_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
 	},
+	{
+		.procname	= "sched_cluster_util_thres_pct",
+		.data		= &sysctl_sched_cluster_util_thres_pct,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_douintvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_INT_MAX,
+	},
+	{
+		.procname	= "sched_idle_enough",
+		.data		= &sysctl_sched_idle_enough,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_douintvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_INT_MAX,
+	},
 	{
 		.procname	= "sched_long_running_rt_task_ms",
 		.data		= &sysctl_sched_long_running_rt_task_ms,

kernel/sched/walt/walt.c

@@ -343,6 +343,98 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
 static void rollover_cpu_window(struct rq *rq, bool full_window);
 static void rollover_top_tasks(struct rq *rq, bool full_window);
 
+/*
+ * if last window's average capacity is less than or
+ * equal to the current capacity, return true.
+ */
+static inline bool is_cpufreq_avg_or_higher(int cpu)
+{
+	unsigned int avg_cap = sched_get_cpu_avg_cap(cpu);
+	unsigned int cur_cap = capacity_curr_of(cpu);
+
+	if (cur_cap >= avg_cap && avg_cap != 0)
+		return true;
+
+	return false;
+}
+
+/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster.
+ * @start_cpu:  The cpu from the cluster to choose from
+ *
+ * If the cluster has a 32bit capable cpu return it regardless
+ * of whether it is halted or not.
+ *
+ * If the cluster does not have a 32 bit capable cpu, find the
+ * first unhalted, active cpu in this cluster.
+ */
+int walt_find_cluster_packing_cpu(int start_cpu)
+{
+	struct rq *rq = cpu_rq(start_cpu);
+	struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1;
+	struct walt_sched_cluster *cluster = wrq->cluster;
+	cpumask_t unhalted_cpus;
+	cpumask_t cluster_32bit_cpus;
+
+	/* find all 32 bit capable cpus in this cluster */
+	cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask());
+
+	/* pack 32 bit and 64 bit tasks on the same cpu, if possible */
+	if (cpumask_weight(&cluster_32bit_cpus) > 0)
+		return cpumask_first(&cluster_32bit_cpus);
+
+	/* find all unhalted active cpus */
+	cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);
+
+	/* find all unhalted active cpus in this cluster */
+	cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);
+
+	/* return the first found unhalted, active cpu, in this cluster */
+	return cpumask_first(&unhalted_cpus);
+}
+
+/* for cfs and rt, determine if packing_cpu should be used */
+bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p)
+{
+	struct rq *rq;
+	struct walt_rq *wrq;
+	struct walt_sched_cluster *cluster;
+
+	/* packing cpu must be a valid cpu for runqueue lookup */
+	if (packing_cpu >= nr_cpu_ids)
+		return false;
+
+	rq = cpu_rq(packing_cpu);
+	wrq = (struct walt_rq *)rq->android_vendor_data1;
+	cluster = wrq->cluster;
+
+	/* if idle_enough feature is not enabled */
+	if (!sysctl_sched_idle_enough)
+		return false;
+
+	/* if cpu is not allowed for this task */
+	if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr))
+		return false;
+
+	/* if cluster util is high */
+	if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct)
+		return false;
+
+	/* if cpu utilization is high */
+	if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough)
+		return false;
+
+	/* don't pack big tasks */
+	if (task_util(p) >= sysctl_sched_idle_enough)
+		return false;
+
+	/* if cpufreq is lower than the previous window */
+	if (!is_cpufreq_avg_or_higher(packing_cpu))
+		return false;
+
+	/* the packing cpu can be used, so pack! */
+	return true;
+}
+
 /*
  * Demand aggregation for frequency purpose:
  *

kernel/sched/walt/walt.h

@@ -187,6 +187,9 @@ extern int max_possible_cluster_id;
 extern unsigned int __read_mostly sched_init_task_load_windows;
 extern unsigned int __read_mostly sched_load_granule;
 
+extern unsigned int sysctl_sched_idle_enough;
+extern unsigned int sysctl_sched_cluster_util_thres_pct;
+
 /* 1ms default for 20ms window size scaled to 1024 */
 extern unsigned int sysctl_sched_min_task_util_for_boost;
 extern unsigned int sysctl_sched_min_task_util_for_uclamp;
@@ -345,6 +348,7 @@ struct sched_avg_stats {
 	int nr_misfit;
 	int nr_max;
 	int nr_scaled;
+	u32 avg_cap;
 };
 
 struct waltgov_callback {
@@ -776,6 +780,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)
 
 extern struct sched_avg_stats *sched_get_nr_running_avg(void);
 extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster);
+extern unsigned int sched_get_cpu_avg_cap(int cpu);
+extern unsigned int waltgov_get_avg_cap(unsigned int cpu);
 extern void sched_update_hyst_times(void);
 
 extern void walt_rt_init(void);
@@ -784,6 +790,8 @@ extern void walt_halt_init(void);
 extern void walt_fixup_init(void);
 extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
 					int sync, int sibling_count_hint);
+extern int walt_find_cluster_packing_cpu(int start_cpu);
+extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p);
 
 static inline unsigned int cpu_max_possible_freq(int cpu)
 {