From 550838ff0a3ba1cffe3447b03b77c42c209954b4 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:48:58 -0700 Subject: [PATCH] sched/walt: Add MVP tasks feature MVP tasks feature is intended for improving general scheduler performance. Change-Id: I7262daf52c3ca4c627642cc5e2aa33a7b33a2305 Signed-off-by: Pavankumar Kondeti Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/walt.h | 4 + kernel/sched/walt/trace.h | 66 ++++++++- kernel/sched/walt/walt.c | 16 ++- kernel/sched/walt/walt.h | 32 +++++ kernel/sched/walt/walt_cfs.c | 264 ++++++++++++++++++++++++++++++++++- kernel/sched/walt/walt_lb.c | 8 +- 6 files changed, 378 insertions(+), 12 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 38df32abd9cc..918466220ba0 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -114,6 +114,10 @@ struct walt_task_struct { cpumask_t cpus_requested; bool iowaited; int prev_on_rq; + struct list_head mvp_list; + u64 sum_exec_snapshot; + u64 total_exec; + int mvp_prio; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 4effae0bbaad..b6f8a8738d3b 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1023,10 +1023,9 @@ TRACE_EVENT(sched_find_best_target, TRACE_EVENT(sched_enq_deq_task, - TP_PROTO(struct task_struct *p, bool enqueue, - unsigned int cpus_allowed), + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed, bool mvp), - TP_ARGS(p, enqueue, cpus_allowed), + TP_ARGS(p, enqueue, cpus_allowed, mvp), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) @@ -1039,7 +1038,8 @@ TRACE_EVENT(sched_enq_deq_task, __field(unsigned int, cpus_allowed) __field(unsigned int, demand) __field(unsigned int, pred_demand) - __field(bool, compat_thread) + __field(bool, compat_thread) + __field(bool, mvp) ), TP_fast_assign( @@ -1054,9 +1054,10 @@ TRACE_EVENT(sched_enq_deq_task, __entry->demand = task_load(p); __entry->pred_demand = task_pl(p); __entry->compat_thread = is_compat_thread(task_thread_info(p)); + __entry->mvp = mvp; ), - TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d", + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d", __entry->cpu, __entry->enqueue ? "enqueue" : "dequeue", __entry->comm, __entry->pid, @@ -1064,7 +1065,7 @@ TRACE_EVENT(sched_enq_deq_task, __entry->rt_nr_running, __entry->cpus_allowed, __entry->demand, __entry->pred_demand, - __entry->compat_thread) + __entry->compat_thread, __entry->mvp) ); TRACE_EVENT(walt_window_rollover, @@ -1083,6 +1084,59 @@ TRACE_EVENT(walt_window_rollover, TP_printk("window_start=%llu", __entry->window_start) ); + +DECLARE_EVENT_CLASS(walt_cfs_mvp_task_template, + + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + + TP_ARGS(p, wts, limit), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, mvp_prio) + __field(int, cpu) + __field(u64, exec) + __field(unsigned int, limit) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->mvp_prio = wts->mvp_prio; + __entry->cpu = task_cpu(p); + __entry->exec = wts->total_exec; + __entry->limit = limit; + ), + + TP_printk("comm=%s pid=%d prio=%d mvp_prio=%d cpu=%d exec=%llu limit=%u", + __entry->comm, __entry->pid, __entry->prio, + __entry->mvp_prio, __entry->cpu, __entry->exec, + __entry->limit) +); + +/* called upon MVP task de-activation. exec will be more than limit */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_deactivate_mvp_task, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP is returned to run next */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_pick_next, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (current) is not preempted by waking task */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_nopreempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (waking task) preempts the current */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_preempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 5ffebd42d99d..02bd142e11c1 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2243,6 +2243,11 @@ static void init_new_task_load(struct task_struct *p) wts->misfit = false; wts->rtg_high_prio = false; wts->unfilter = sysctl_sched_task_unfilter_period; + + INIT_LIST_HEAD(&wts->mvp_list); + wts->sum_exec_snapshot = 0; + wts->total_exec = 0; + wts->mvp_prio = WALT_NOT_MVP; } static void init_existing_task_load(struct task_struct *p) @@ -3621,6 +3626,8 @@ static void walt_sched_init_rq(struct rq *rq) clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]); } wrq->notif_pending = false; + + INIT_LIST_HEAD(&wrq->mvp_tasks); } void sched_window_nr_ticks_change(void) @@ -3814,10 +3821,11 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (walt_fair_task(p)) { wts->misfit = !task_fits_max(p, rq->cpu); inc_rq_walt_stats(rq, p); + walt_cfs_enqueue_task(rq, p); } walt_inc_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]); + trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) @@ -3841,11 +3849,13 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st sched_update_nr_prod(rq->cpu, -1); - if (walt_fair_task(p)) + if (walt_fair_task(p)) { dec_rq_walt_stats(rq, p); + walt_cfs_dequeue_task(rq, p); + } walt_dec_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]); + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } static void android_rvh_update_misfit_status(void *unused, struct task_struct *p, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1c5cd2288fff..ab3584f9e7a3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -118,6 +118,7 @@ struct walt_rq { bool high_irqload; u64 last_cc_update; u64 cycles; + struct list_head mvp_tasks; }; struct walt_sched_cluster { @@ -729,6 +730,22 @@ static inline bool walt_low_latency_task(struct task_struct *p) (task_util(p) < sysctl_walt_low_latency_task_threshold); } +static inline bool walt_binder_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_procfs_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) { struct cpuidle_state *idle = idle_get_state(rq); @@ -861,4 +878,19 @@ static inline bool walt_fair_task(struct task_struct *p) { return p->prio >= MAX_RT_PRIO && !is_idle_task(p); } + + +#define WALT_MVP_SLICE 3000000U +#define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) + +#define WALT_RTG_MVP 0 +#define WALT_BINDER_MVP 1 + +#define WALT_NOT_MVP -1 + +#define is_mvp(wts) (wts->mvp_prio != WALT_NOT_MVP) +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_tick(struct rq *rq); + #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 0677b7a90b3c..7b424d3b1d43 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -689,7 +689,8 @@ static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vrunti } } -static void walt_binder_low_latency_set(void *unused, struct task_struct *task) +static void walt_binder_low_latency_set(void *unused, struct task_struct *task, + bool sync, struct binder_proc *proc) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; @@ -738,16 +739,275 @@ static void binder_restore_priority_hook(void *data, if (wts->boost == TASK_BOOST_STRICT_MAX) wts->boost = bndrtrans->android_vendor_data1; + +} +/* + * Higher prio mvp can preempt lower prio mvp. + * + * However, the lower prio MVP slice will be more since we expect them to + * be the work horses. For example, binders will have higher prio MVP and + * they can preempt long running rtg prio tasks but binders loose their + * powers with in 3 msec where as rtg prio tasks can run more than that. + */ +static inline int walt_get_mvp_task_prio(struct task_struct *p) +{ + if ((per_task_boost(p) == TASK_BOOST_STRICT_MAX) || + task_rtg_high_prio(p) || + walt_procfs_low_latency_task(p)) + return WALT_RTG_MVP; + + if (walt_binder_low_latency_task(p)) + return WALT_BINDER_MVP; + + return WALT_NOT_MVP; +} + +static inline unsigned int walt_cfs_mvp_task_limit(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Binder MVP tasks are high prio but have only single slice */ + if (wts->mvp_prio == WALT_BINDER_MVP) + return WALT_MVP_SLICE; + + return WALT_MVP_LIMIT; +} + +static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struct *wts, + bool at_front) +{ + struct list_head *pos; + + list_for_each(pos, &wrq->mvp_tasks) { + struct walt_task_struct *tmp_wts = container_of(pos, struct walt_task_struct, + mvp_list); + + if (at_front) { + if (wts->mvp_prio >= tmp_wts->mvp_prio) + break; + } else { + if (wts->mvp_prio > tmp_wts->mvp_prio) + break; + } + } + + list_add(&wts->mvp_list, pos->prev); +} + +static void walt_cfs_deactivate_mvp_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + list_del_init(&wts->mvp_list); + wts->mvp_prio = WALT_NOT_MVP; + + /* + * Reset the exec time during sleep so that it starts + * from scratch upon next wakeup. total_exec should + * be preserved when task is enq/deq while it is on + * runqueue. + */ + if (p->state != TASK_RUNNING) + wts->total_exec = 0; +} + +/* + * MVP task runtime update happens here. Three possibilities: + * + * de-activated: The MVP consumed its runtime. Non MVP can preempt. + * slice expired: MVP slice is expired and other MVP can preempt. + * slice not expired: This MVP task can continue to run. + */ +static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; + s64 delta; + unsigned int limit; + + /* sum_exec_snapshot can be ahead. See below increment */ + delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot; + if (delta < 0) + delta = 0; + else + delta += rq_clock_task(rq) - curr->se.exec_start; + + /* slice is not expired */ + if (delta < WALT_MVP_SLICE) + return; + + /* + * slice is expired, check if we have to deactivate the + * MVP task, otherwise requeue the task in the list so + * that other MVP tasks gets a chance. + */ + wts->sum_exec_snapshot += delta; + wts->total_exec += delta; + + limit = walt_cfs_mvp_task_limit(curr); + if (wts->total_exec > limit) { + walt_cfs_deactivate_mvp_task(curr); + trace_walt_cfs_deactivate_mvp_task(curr, wts, limit); + return; + } + + /* slice expired. re-queue the task */ + list_del(&wts->mvp_list); + walt_cfs_insert_mvp_task(wrq, wts, false); +} + +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int mvp_prio = walt_get_mvp_task_prio(p); + + if (mvp_prio == WALT_NOT_MVP) + return; + + /* + * This can happen during migration or enq/deq for prio/class change. + * it was once MVP but got demoted, it will not be MVP until + * it goes to sleep again. + */ + if (wts->total_exec > walt_cfs_mvp_task_limit(p)) + return; + + wts->mvp_prio = mvp_prio; + walt_cfs_insert_mvp_task(wrq, wts, task_running(rq, p)); + + /* + * We inserted the task at the appropriate position. Take the + * task runtime snapshot. From now onwards we use this point as a + * baseline to enforce the slice and demotion. + */ + if (!wts->total_exec) /* queue after sleep */ + wts->sum_exec_snapshot = p->se.sum_exec_runtime; + +} + +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!list_empty(&wts->mvp_list)) + walt_cfs_deactivate_mvp_task(p); +} + +void walt_cfs_tick(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (list_empty(&wts->mvp_list)) + return; + + walt_cfs_account_mvp_runtime(rq, rq->curr); + /* + * If the current is not MVP means, we have to re-schedule to + * see if we can run any other task including MVP tasks. + */ + if ((wrq->mvp_tasks.next != &wts->mvp_list) && rq->cfs.h_nr_running > 1) + resched_curr(rq); +} + +/* + * When preempt = false and nopreempt = false, we leave the preemption + * decision to CFS. + */ +static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct task_struct *p, + bool *preempt, bool *nopreempt, int wake_flags, + struct sched_entity *se, struct sched_entity *pse, + int next_buddy_marked, unsigned int granularity) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1; + struct task_struct *c = rq->curr; + struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1; + bool resched = false; + bool p_is_mvp, curr_is_mvp; + + if (unlikely(walt_disabled)) + return; + + p_is_mvp = !list_empty(&wts_p->mvp_list); + curr_is_mvp = !list_empty(&wts_c->mvp_list); + + /* + * current is not MVP, so preemption decision + * is simple. + */ + if (!curr_is_mvp) { + if (p_is_mvp) + goto preempt; + return; /* CFS decides preemption */ + } + + /* + * current is MVP. update its runtime before deciding the + * preemption. + */ + walt_cfs_account_mvp_runtime(rq, c); + resched = (wrq->mvp_tasks.next != &wts_c->mvp_list); + + /* + * current is no longer eligible to run. It must have been + * picked (because of MVP) ahead of other tasks in the CFS + * tree, so drive preemption to pick up the next task from + * the tree, which also includes picking up the first in + * the MVP queue. + */ + if (resched) + goto preempt; + + /* current is the first in the queue, so no preemption */ + *nopreempt = true; + trace_walt_cfs_mvp_wakeup_nopreempt(c, wts_c, walt_cfs_mvp_task_limit(c)); + return; +preempt: + *preempt = true; + trace_walt_cfs_mvp_wakeup_preempt(p, wts_p, walt_cfs_mvp_task_limit(p)); +} + +static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct task_struct **p, + struct sched_entity **se, bool *repick, bool simple, + struct task_struct *prev) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts; + struct task_struct *mvp; + + if (unlikely(walt_disabled)) + return; + + /* We don't have MVP tasks queued */ + if (list_empty(&wrq->mvp_tasks)) + return; + + /* Return the first task from MVP queue */ + wts = list_first_entry(&wrq->mvp_tasks, struct walt_task_struct, mvp_list); + mvp = wts_to_ts(wts); + + *p = mvp; + *se = &mvp->se; + *repick = true; + + trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); } void walt_cfs_init(void) { register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); - register_trace_android_rvh_place_entity(walt_place_entity, NULL); register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL); register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL); + + register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL); + register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL); } diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 960da0dca5d4..921530c441ce 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -494,7 +494,13 @@ static void walt_lb_tick(void *unused, struct rq *rq) if (unlikely(walt_disabled)) return; - if (!rq->misfit_task_load || !walt_fair_task(p)) + + if (!walt_fair_task(p)) + return; + + walt_cfs_tick(rq); + + if (!rq->misfit_task_load) return; if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1)