sched/walt: Add MVP tasks feature

MVP tasks feature is intended for improving general scheduler
performance.

Change-Id: I7262daf52c3ca4c627642cc5e2aa33a7b33a2305
Signed-off-by: Pavankumar Kondeti <pkondeti@codeaurora.org>
Signed-off-by: Sai Harshini Nimmala <snimmala@codeaurora.org>

include/linux/sched/walt.h

@@ -114,6 +114,10 @@ struct walt_task_struct {
 	cpumask_t			cpus_requested;
 	bool				iowaited;
 	int				prev_on_rq;
+	struct list_head		mvp_list;
+	u64				sum_exec_snapshot;
+	u64				total_exec;
+	int				mvp_prio;
 };
 
 #define wts_to_ts(wts) ({ \

kernel/sched/walt/trace.h

@@ -1023,10 +1023,9 @@ TRACE_EVENT(sched_find_best_target,
 
 TRACE_EVENT(sched_enq_deq_task,
 
-	TP_PROTO(struct task_struct *p, bool enqueue,
-				unsigned int cpus_allowed),
+	TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed, bool mvp),
 
-	TP_ARGS(p, enqueue, cpus_allowed),
+	TP_ARGS(p, enqueue, cpus_allowed, mvp),
 
 	TP_STRUCT__entry(
 		__array(char,		comm, TASK_COMM_LEN)
@@ -1039,7 +1038,8 @@ TRACE_EVENT(sched_enq_deq_task,
 		__field(unsigned int,	cpus_allowed)
 		__field(unsigned int,	demand)
 		__field(unsigned int,	pred_demand)
-		__field(bool,		 compat_thread)
+		__field(bool,		compat_thread)
+		__field(bool,		mvp)
 	),
 
 	TP_fast_assign(
@@ -1054,9 +1054,10 @@ TRACE_EVENT(sched_enq_deq_task,
 		__entry->demand		= task_load(p);
 		__entry->pred_demand	= task_pl(p);
 		__entry->compat_thread	= is_compat_thread(task_thread_info(p));
+		__entry->mvp		= mvp;
 	),
 
-	TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d",
+	TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d",
 			__entry->cpu,
 			__entry->enqueue ? "enqueue" : "dequeue",
 			__entry->comm, __entry->pid,
@@ -1064,7 +1065,7 @@ TRACE_EVENT(sched_enq_deq_task,
 			__entry->rt_nr_running,
 			__entry->cpus_allowed, __entry->demand,
 			__entry->pred_demand,
-			__entry->compat_thread)
+			__entry->compat_thread, __entry->mvp)
 );
 
 TRACE_EVENT(walt_window_rollover,
@@ -1083,6 +1084,59 @@ TRACE_EVENT(walt_window_rollover,
 
 	TP_printk("window_start=%llu", __entry->window_start)
 );
+
+DECLARE_EVENT_CLASS(walt_cfs_mvp_task_template,
+
+	TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit),
+
+	TP_ARGS(p, wts, limit),
+
+	TP_STRUCT__entry(
+		__array(char,		comm,	TASK_COMM_LEN)
+		__field(pid_t,		pid)
+		__field(int,		prio)
+		__field(int,		mvp_prio)
+		__field(int,		cpu)
+		__field(u64,		exec)
+		__field(unsigned int,	limit)
+	),
+
+	TP_fast_assign(
+		memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
+		__entry->pid		= p->pid;
+		__entry->prio		= p->prio;
+		__entry->mvp_prio	= wts->mvp_prio;
+		__entry->cpu		= task_cpu(p);
+		__entry->exec		= wts->total_exec;
+		__entry->limit		= limit;
+	),
+
+	TP_printk("comm=%s pid=%d prio=%d mvp_prio=%d cpu=%d exec=%llu limit=%u",
+		__entry->comm, __entry->pid, __entry->prio,
+		__entry->mvp_prio, __entry->cpu, __entry->exec,
+		__entry->limit)
+);
+
+/* called upon MVP task de-activation. exec will be more than limit */
+DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_deactivate_mvp_task,
+	     TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit),
+	     TP_ARGS(p, wts, limit));
+
+/* called upon when MVP is returned to run next */
+DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_pick_next,
+	     TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit),
+	     TP_ARGS(p, wts, limit));
+
+/* called upon when MVP (current) is not preempted by waking task */
+DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_nopreempt,
+	     TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit),
+	     TP_ARGS(p, wts, limit));
+
+/* called upon when MVP (waking task) preempts the current */
+DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_preempt,
+	     TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit),
+	     TP_ARGS(p, wts, limit));
+
 #endif /* _TRACE_WALT_H */
 
 #undef TRACE_INCLUDE_PATH

kernel/sched/walt/walt.c

@@ -2243,6 +2243,11 @@ static void init_new_task_load(struct task_struct *p)
 	wts->misfit = false;
 	wts->rtg_high_prio = false;
 	wts->unfilter = sysctl_sched_task_unfilter_period;
+
+	INIT_LIST_HEAD(&wts->mvp_list);
+	wts->sum_exec_snapshot = 0;
+	wts->total_exec = 0;
+	wts->mvp_prio = WALT_NOT_MVP;
 }
 
 static void init_existing_task_load(struct task_struct *p)
@@ -3621,6 +3626,8 @@ static void walt_sched_init_rq(struct rq *rq)
 		clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]);
 	}
 	wrq->notif_pending = false;
+
+	INIT_LIST_HEAD(&wrq->mvp_tasks);
 }
 
 void sched_window_nr_ticks_change(void)
@@ -3814,10 +3821,11 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st
 	if (walt_fair_task(p)) {
 		wts->misfit = !task_fits_max(p, rq->cpu);
 		inc_rq_walt_stats(rq, p);
+		walt_cfs_enqueue_task(rq, p);
 	}
 
 	walt_inc_cumulative_runnable_avg(rq, p);
-	trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]);
+	trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts));
 }
 
 static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags)
@@ -3841,11 +3849,13 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st
 
 	sched_update_nr_prod(rq->cpu, -1);
 
-	if (walt_fair_task(p))
+	if (walt_fair_task(p)) {
 		dec_rq_walt_stats(rq, p);
+		walt_cfs_dequeue_task(rq, p);
+	}
 
 	walt_dec_cumulative_runnable_avg(rq, p);
-	trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]);
+	trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts));
 }
 
 static void android_rvh_update_misfit_status(void *unused, struct task_struct *p,

kernel/sched/walt/walt.h

@@ -118,6 +118,7 @@ struct walt_rq {
 	bool			high_irqload;
 	u64			last_cc_update;
 	u64			cycles;
+	struct list_head	mvp_tasks;
 };
 
 struct walt_sched_cluster {
@@ -729,6 +730,22 @@ static inline bool walt_low_latency_task(struct task_struct *p)
 		(task_util(p) < sysctl_walt_low_latency_task_threshold);
 }
 
+static inline bool walt_binder_low_latency_task(struct task_struct *p)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	return (wts->low_latency & WALT_LOW_LATENCY_BINDER) &&
+		(task_util(p) < sysctl_walt_low_latency_task_threshold);
+}
+
+static inline bool walt_procfs_low_latency_task(struct task_struct *p)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) &&
+		(task_util(p) < sysctl_walt_low_latency_task_threshold);
+}
+
 static inline unsigned int walt_get_idle_exit_latency(struct rq *rq)
 {
 	struct cpuidle_state *idle = idle_get_state(rq);
@@ -861,4 +878,19 @@ static inline bool walt_fair_task(struct task_struct *p)
 {
 	return p->prio >= MAX_RT_PRIO && !is_idle_task(p);
 }
+
+
+#define WALT_MVP_SLICE		3000000U
+#define WALT_MVP_LIMIT		(4 * WALT_MVP_SLICE)
+
+#define WALT_RTG_MVP		0
+#define WALT_BINDER_MVP		1
+
+#define WALT_NOT_MVP		-1
+
+#define is_mvp(wts) (wts->mvp_prio != WALT_NOT_MVP)
+void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p);
+void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p);
+void walt_cfs_tick(struct rq *rq);
+
 #endif /* _WALT_H */

kernel/sched/walt/walt_cfs.c

@@ -689,7 +689,8 @@ static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vrunti
 	}
 }
 
-static void walt_binder_low_latency_set(void *unused, struct task_struct *task)
+static void walt_binder_low_latency_set(void *unused, struct task_struct *task,
+					bool sync, struct binder_proc *proc)
 {
 	struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1;
 
@@ -738,16 +739,275 @@ static void binder_restore_priority_hook(void *data,
 
 	if (wts->boost == TASK_BOOST_STRICT_MAX)
 		wts->boost = bndrtrans->android_vendor_data1;
+
+}
+/*
+ * Higher prio mvp can preempt lower prio mvp.
+ *
+ * However, the lower prio MVP slice will be more since we expect them to
+ * be the work horses. For example, binders will have higher prio MVP and
+ * they can preempt long running rtg prio tasks but binders loose their
+ * powers with in 3 msec where as rtg prio tasks can run more than that.
+ */
+static inline int walt_get_mvp_task_prio(struct task_struct *p)
+{
+	if ((per_task_boost(p) == TASK_BOOST_STRICT_MAX) ||
+		task_rtg_high_prio(p) ||
+		walt_procfs_low_latency_task(p))
+		return WALT_RTG_MVP;
+
+	if (walt_binder_low_latency_task(p))
+		return WALT_BINDER_MVP;
+
+	return WALT_NOT_MVP;
+}
+
+static inline unsigned int walt_cfs_mvp_task_limit(struct task_struct *p)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	/* Binder MVP tasks are high prio but have only single slice */
+	if (wts->mvp_prio == WALT_BINDER_MVP)
+		return WALT_MVP_SLICE;
+
+	return WALT_MVP_LIMIT;
+}
+
+static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struct *wts,
+				     bool at_front)
+{
+	struct list_head *pos;
+
+	list_for_each(pos, &wrq->mvp_tasks) {
+		struct walt_task_struct *tmp_wts = container_of(pos, struct walt_task_struct,
+								mvp_list);
+
+		if (at_front) {
+			if (wts->mvp_prio >= tmp_wts->mvp_prio)
+				break;
+		} else {
+			if (wts->mvp_prio > tmp_wts->mvp_prio)
+				break;
+		}
+	}
+
+	list_add(&wts->mvp_list, pos->prev);
+}
+
+static void walt_cfs_deactivate_mvp_task(struct task_struct *p)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	list_del_init(&wts->mvp_list);
+	wts->mvp_prio = WALT_NOT_MVP;
+
+	/*
+	 * Reset the exec time during sleep so that it starts
+	 * from scratch upon next wakeup. total_exec should
+	 * be preserved when task is enq/deq while it is on
+	 * runqueue.
+	 */
+	if (p->state != TASK_RUNNING)
+		wts->total_exec = 0;
+}
+
+/*
+ * MVP task runtime update happens here. Three possibilities:
+ *
+ * de-activated: The MVP consumed its runtime. Non MVP can preempt.
+ * slice expired: MVP slice is expired and other MVP can preempt.
+ * slice not expired: This MVP task can continue to run.
+ */
+static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr)
+{
+	struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+	struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1;
+	s64 delta;
+	unsigned int limit;
+
+	/* sum_exec_snapshot can be ahead. See below increment */
+	delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot;
+	if (delta < 0)
+		delta = 0;
+	else
+		delta += rq_clock_task(rq) - curr->se.exec_start;
+
+	/* slice is not expired */
+	if (delta < WALT_MVP_SLICE)
+		return;
+
+	/*
+	 * slice is expired, check if we have to deactivate the
+	 * MVP task, otherwise requeue the task in the list so
+	 * that other MVP tasks gets a chance.
+	 */
+	wts->sum_exec_snapshot += delta;
+	wts->total_exec += delta;
+
+	limit = walt_cfs_mvp_task_limit(curr);
+	if (wts->total_exec > limit) {
+		walt_cfs_deactivate_mvp_task(curr);
+		trace_walt_cfs_deactivate_mvp_task(curr, wts, limit);
+		return;
+	}
+
+	/* slice expired. re-queue the task */
+	list_del(&wts->mvp_list);
+	walt_cfs_insert_mvp_task(wrq, wts, false);
+}
+
+void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p)
+{
+	struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+	int mvp_prio = walt_get_mvp_task_prio(p);
+
+	if (mvp_prio == WALT_NOT_MVP)
+		return;
+
+	/*
+	 * This can happen during migration or enq/deq for prio/class change.
+	 * it was once MVP but got demoted, it will not be MVP until
+	 * it goes to sleep again.
+	 */
+	if (wts->total_exec > walt_cfs_mvp_task_limit(p))
+		return;
+
+	wts->mvp_prio = mvp_prio;
+	walt_cfs_insert_mvp_task(wrq, wts, task_running(rq, p));
+
+	/*
+	 * We inserted the task at the appropriate position. Take the
+	 * task runtime snapshot. From now onwards we use this point as a
+	 * baseline to enforce the slice and demotion.
+	 */
+	if (!wts->total_exec) /* queue after sleep */
+		wts->sum_exec_snapshot = p->se.sum_exec_runtime;
+
+}
+
+void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	if (!list_empty(&wts->mvp_list))
+		walt_cfs_deactivate_mvp_task(p);
+}
+
+void walt_cfs_tick(struct rq *rq)
+{
+	struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+	struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1;
+
+	if (unlikely(walt_disabled))
+		return;
+
+	if (list_empty(&wts->mvp_list))
+		return;
+
+	walt_cfs_account_mvp_runtime(rq, rq->curr);
+	/*
+	 * If the current is not MVP means, we have to re-schedule to
+	 * see if we can run any other task including MVP tasks.
+	 */
+	if ((wrq->mvp_tasks.next != &wts->mvp_list) && rq->cfs.h_nr_running > 1)
+		resched_curr(rq);
+}
+
+/*
+ * When preempt = false and nopreempt = false, we leave the preemption
+ * decision to CFS.
+ */
+static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct task_struct *p,
+					  bool *preempt, bool *nopreempt, int wake_flags,
+					  struct sched_entity *se, struct sched_entity *pse,
+					  int next_buddy_marked, unsigned int granularity)
+{
+	struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+	struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1;
+	struct task_struct *c = rq->curr;
+	struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1;
+	bool resched = false;
+	bool p_is_mvp, curr_is_mvp;
+
+	if (unlikely(walt_disabled))
+		return;
+
+	p_is_mvp = !list_empty(&wts_p->mvp_list);
+	curr_is_mvp = !list_empty(&wts_c->mvp_list);
+
+	/*
+	 * current is not MVP, so preemption decision
+	 * is simple.
+	 */
+	if (!curr_is_mvp) {
+		if (p_is_mvp)
+			goto preempt;
+		return; /* CFS decides preemption */
+	}
+
+	/*
+	 * current is MVP. update its runtime before deciding the
+	 * preemption.
+	 */
+	walt_cfs_account_mvp_runtime(rq, c);
+	resched = (wrq->mvp_tasks.next != &wts_c->mvp_list);
+
+	/*
+	 * current is no longer eligible to run. It must have been
+	 * picked (because of MVP) ahead of other tasks in the CFS
+	 * tree, so drive preemption to pick up the next task from
+	 * the tree, which also includes picking up the first in
+	 * the MVP queue.
+	 */
+	if (resched)
+		goto preempt;
+
+	/* current is the first in the queue, so no preemption */
+	*nopreempt = true;
+	trace_walt_cfs_mvp_wakeup_nopreempt(c, wts_c, walt_cfs_mvp_task_limit(c));
+	return;
+preempt:
+	*preempt = true;
+	trace_walt_cfs_mvp_wakeup_preempt(p, wts_p, walt_cfs_mvp_task_limit(p));
+}
+
+static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct task_struct **p,
+					    struct sched_entity **se, bool *repick, bool simple,
+					    struct task_struct *prev)
+{
+	struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
+	struct walt_task_struct *wts;
+	struct task_struct *mvp;
+
+	if (unlikely(walt_disabled))
+		return;
+
+	/* We don't have MVP tasks queued */
+	if (list_empty(&wrq->mvp_tasks))
+		return;
+
+	/* Return the first task from MVP queue */
+	wts = list_first_entry(&wrq->mvp_tasks, struct walt_task_struct, mvp_list);
+	mvp = wts_to_ts(wts);
+
+	*p = mvp;
+	*se = &mvp->se;
+	*repick = true;
+
+	trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp));
 }
 
 void walt_cfs_init(void)
 {
 	register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL);
-	register_trace_android_rvh_place_entity(walt_place_entity, NULL);
 
 	register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL);
 	register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL);
 
 	register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL);
 	register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL);
+
+	register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL);
+	register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL);
 }

kernel/sched/walt/walt_lb.c

@@ -494,7 +494,13 @@ static void walt_lb_tick(void *unused, struct rq *rq)
 
 	if (unlikely(walt_disabled))
 		return;
-	if (!rq->misfit_task_load || !walt_fair_task(p))
+
+	if (!walt_fair_task(p))
+		return;
+
+	walt_cfs_tick(rq);
+
+	if (!rq->misfit_task_load)
 		return;
 
 	if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1)