sched/walt/avg: cache window based statistics

Previously it was possible to update core control with new
statistics and those statistics would be unavailable to
any other decisions that need to be made.

Update sched_avg to cache the statistics locally and simply
pass a reference to that information back to core control.
Provide a cluster util function to provide window based load.
Create a sysctl node and threshold to use to compare against
cluster util.

Change-Id: I8cc40603e49e29e7f8a343f6ff667ce9abe09cb5
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>

kernel/sched/walt/core_ctl.c

@@ -482,7 +482,7 @@ static struct kobj_type ktype_core_ctl = {
 
 /* ==================== runqueue based core count =================== */
 
-static struct sched_avg_stats nr_stats[WALT_NR_CPUS];
+static struct sched_avg_stats *nr_stats;
 
 /*
  * nr_need:
@@ -686,7 +686,7 @@ static void update_running_avg(void)
 	unsigned long flags;
 	int big_avg = 0;
 
-	sched_get_nr_running_avg(nr_stats);
+	nr_stats = sched_get_nr_running_avg();
 
 	spin_lock_irqsave(&state_lock, flags);
 	for_each_cluster(cluster, index) {
@@ -954,6 +954,14 @@ static void core_ctl_call_notifier(void)
 	atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata);
 }
 
+/*
+ * sched_get_nr_running_avg will wipe out previous statistics and
+ * update it to the values computed since the last call.
+ *
+ * core_ctl_check assumes that the statistics are stable, hence
+ * window based. Therefore core_ctl_check must only be called from
+ * window rollover, or walt_irq_work for not migration.
+ */
 void core_ctl_check(u64 window_start)
 {
 	int cpu;
@@ -1330,6 +1338,8 @@ int core_ctl_init(void)
 
 	spin_lock_init(&core_ctl_pending_lock);
 
+	nr_stats = sched_get_nr_running_avg();
+
 	/* initialize our single kthread, after spin lock init */
 	core_ctl_thread = kthread_run(try_core_ctl, NULL, "core_ctl");
 

kernel/sched/walt/sched_avg.c

@@ -34,6 +34,28 @@ static DEFINE_PER_CPU(u64, util_hyst_time);
 #define NR_THRESHOLD_PCT		40
 #define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC)
 
+struct sched_avg_stats stats[WALT_NR_CPUS];
+unsigned int cstats_util_pct[MAX_CLUSTERS];
+
+/**
+ * sched_get_cluster_util_pct
+ * @return: provide the percentage of this cluter that was used in the
+ *          previous window.
+ *
+ * This routine may be called any number of times as needed during
+ * a window, but will always return the same result until window
+ * rollover.
+ */
+unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster)
+{
+	unsigned int cluster_util_pct = 0;
+
+	if (cluster->id < MAX_CLUSTERS)
+		cluster_util_pct = cstats_util_pct[cluster->id];
+
+	return cluster_util_pct;
+}
+
 /**
  * sched_get_nr_running_avg
  * @return: Average nr_running, iowait and nr_big_tasks value since last poll.
@@ -41,18 +63,22 @@ static DEFINE_PER_CPU(u64, util_hyst_time);
  *	    of accuracy.
  *
  * Obtains the average nr_running value since the last poll.
- * This function may not be called concurrently with itself
+ * This function may not be called concurrently with itself.
+ *
+ * It is assumed that this function is called at most once per window
+ * rollover.
  */
-void sched_get_nr_running_avg(struct sched_avg_stats *stats)
+struct sched_avg_stats *sched_get_nr_running_avg(void)
 {
 	int cpu;
 	u64 curr_time = sched_clock();
 	u64 period = curr_time - last_get_time;
 	u64 tmp_nr, tmp_misfit;
 	bool any_hyst_time = false;
+	struct walt_sched_cluster *cluster;
 
 	if (!period)
-		return;
+		goto done;
 
 	/* read and reset nr_running counts */
 	for_each_possible_cpu(cpu) {
@@ -96,6 +122,27 @@ void sched_get_nr_running_avg(struct sched_avg_stats *stats)
 		spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
 	}
 
+	/* collect cluster load stats */
+	for_each_sched_cluster(cluster) {
+		unsigned int num_cpus = cpumask_weight(&cluster->cpus);
+		unsigned int sum_util_pct = 0;
+
+		/* load is already scaled, see freq_policy_load/prev_runnable_sum */
+		for_each_cpu(cpu, &cluster->cpus) {
+			struct rq *rq = cpu_rq(cpu);
+			struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+
+			/* compute the % this cpu's utilization of the cpu capacity,
+			 * and sum it across all cpus
+			 */
+			sum_util_pct +=
+				(wrq->util * 100) / arch_scale_cpu_capacity(cpu);
+		}
+
+		/* calculate the averge per-cpu utilization */
+		cstats_util_pct[cluster->id] = sum_util_pct / num_cpus;
+	}
+
 	for_each_possible_cpu(cpu) {
 		if (per_cpu(coloc_hyst_time, cpu)) {
 			any_hyst_time = true;
@@ -107,6 +154,8 @@ void sched_get_nr_running_avg(struct sched_avg_stats *stats)
 
 	last_get_time = curr_time;
 
+done:
+	return &stats[0];
 }
 EXPORT_SYMBOL(sched_get_nr_running_avg);
 

kernel/sched/walt/walt.c

@@ -653,6 +653,12 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea
 
 	util = scale_time_to_util(freq_policy_load(rq, reason));
 
+	/*
+	 * util is on a scale of 0 to 1024.  this is the utilization
+	 * of the cpu in the last window
+	 */
+	wrq->util = util;
+
 	if (walt_load) {
 		u64 nl = wrq->nt_prev_runnable_sum +
 				wrq->grp_time.nt_prev_runnable_sum;

kernel/sched/walt/walt.h

@@ -120,6 +120,7 @@ struct walt_rq {
 	bool			high_irqload;
 	u64			last_cc_update;
 	u64			cycles;
+	u64			util;
 	struct list_head	mvp_tasks;
 	int                     num_mvp_tasks;
 	u64			latest_clock;
@@ -773,7 +774,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)
 	return task_fits_capacity(p, capacity, cpu);
 }
 
-extern void sched_get_nr_running_avg(struct sched_avg_stats *stats);
+extern struct sched_avg_stats *sched_get_nr_running_avg(void);
+extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster);
 extern void sched_update_hyst_times(void);
 
 extern void walt_rt_init(void);