From 327df46119a36764fef5baa3d631cd52d6344146 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 21 Mar 2022 11:27:51 -0700 Subject: [PATCH] sched/walt: Optimize update_task_ravg() calls When the window is advanced to the next boundary, the CPU busy time trackers need to be advanced and top-task trackers should be flipped. This rollover needs to be done just once for a cpu per window advancement, while it could be subject to numerous update_task_ravg (called utra henceforth) with various tasks spanning window cross over point. The current code achieves this one-time-rollover only when it sees a utra called for the currently running task and its time has crossed over in the new window. As a result of this rollover being done only when utra is called for current task, it became mandatory to call utra on current task before calling utra on a non-current task. This ensured a proper window rollover i.e. the trackers reset/flipped if needed prior to calling utra on non-current task which may have entered its execution in a new window. Explaining pictorially old new ws ms ws wc | | | | V V V V -----|----------------|-------- prev curr In the above example, a non current task wc may have entered the new window but prev and curr are still set as per the old window start. The time from new ws to wc cannot be accounted because curr(_runnable_sum) hasnt advanced i.e hasn't rolled over. Hence utra is called with the current task which causes a rollover i.e. it advances prev, curr(_runnable_sum) pointers amongst other things old new ws ms ws wc | | | | V V V V -----|----------------|-------- prev curr We see utra on current being called, when a task 'p' is woken up, two update_task_ravg() calls are needed. update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); when a task 'p' is migrating, three update_task_ravg() calls are needed. update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, wallclock, 0); update_task_ravg(dest_rq->curr, dest_rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); when a task 'p' is tranferring in/out of rtg update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); We can optimize the number of update_task_ravg() calls by allowing to rollover regardless of utra on current task. The best place to do it is in update_window_start() right along the code which advances the window start. Since the update is not happening on the running task i.e current, the CPU utilization i.e prev_runnable_sum does not reflect the current task utilization immediately after the window is rolled over. However update_task_ravg() is called on the current before presenting the utilization to the governor. So there should not be any impact on the final frequency selection. A TASK_WAKE of a iowaited task needs to terminate the accumulation of iowait time on the cpu where the task had slept. The mark_start of the idle task is advanced and the "advanced" time is accounted as cpu busy time. For this situation alone call utra on idle task if an iowaited task is waking up. Change-Id: Ifb771b41bd62f3c748035bf1e5e5ad7912b400f4 Signed-off-by: Pavankumar Kondeti Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 70 ++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2496935cc2fe..8a57b2c96f52 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -342,6 +342,9 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, pred_demand_delta); } +static void rollover_cpu_window(struct rq *rq, bool full_window); +static void rollover_top_tasks(struct rq *rq, bool full_window); + /* * Demand aggregation for frequency purpose: * @@ -394,6 +397,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) int nr_windows; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; u64 old_window_start = wrq->window_start; + bool full_window; if (wallclock < wrq->latest_clock) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than latest_clock=%llu(0x%llx)", @@ -417,6 +421,10 @@ update_window_start(struct rq *rq, u64 wallclock, int event) wrq->prev_window_size = sched_ravg_window; + full_window = nr_windows > 1; + rollover_cpu_window(rq, full_window); + rollover_top_tasks(rq, full_window); + return old_window_start; } @@ -1010,14 +1018,20 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); - walt_update_task_ravg(task_rq(p)->curr, task_rq(p), - TASK_UPDATE, - wallclock, 0); - walt_update_task_ravg(dest_rq->curr, dest_rq, - TASK_UPDATE, wallclock, 0); - - walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, - wallclock, 0); + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); + /* + * The above update might have rolled over the + * window for this migrating task. Since we are + * going to adjust the destination CPU's busy time + * counters with the task busytime counters, roll over + * the window for the destination CPU also. + * + * The update_window_start() does nothing if the window + * is not rolled over, so there is no need to check for + * window boundary or if the counters will be accessed + * or not. + */ + update_window_start(dest_rq, wallclock, TASK_UPDATE); update_task_cpu_cycles(p, new_cpu, wallclock); @@ -1586,11 +1600,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, new_task = is_new_task(p); - if (p_is_curr_task && new_window) { - rollover_cpu_window(rq, full_window); - rollover_top_tasks(rq, full_window); - } - if (!account_busy_for_cpu_time(rq, p, irqtime, event)) goto done; @@ -1631,15 +1640,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, goto done; } + /* + * situations below this need window rollover, which should already be done + * in update_window_start() + */ + if (!p_is_curr_task) { /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has also started, but p is not the current task, so the - * window is not rolled over - just split up and account - * as necessary into curr and prev. The window is only - * rolled over when a new window is processed for the current - * task. + * must have been started in udpate_window_start() + * - just split up and account as necessary into curr and prev. * * Irqtime can't be accounted by a task that isn't the * currently running task. @@ -1684,8 +1695,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. If any of these three above conditions are true + * must have been started in udpate_window_start() + * If any of these three above conditions are true * then this busy time can't be accounted as irqtime. * * Busy time for the idle task need not be accounted. @@ -1717,10 +1728,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, } } - /* - * Rollover is done here by overwriting the values in - * prev_runnable_sum and curr_runnable_sum. - */ *prev_runnable_sum += delta; if (new_task) *nt_prev_runnable_sum += delta; @@ -1743,8 +1750,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. The current task must be the idle task because + * must have been started in udpate_window_start() + * The current task must be the idle task because * irqtime is not accounted for any other task. * * Irqtime will be accounted each time we process IRQ activity @@ -1756,11 +1763,11 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, mark_start = wallclock - irqtime; /* - * Roll window over. If IRQ busy time was just in the current + * If IRQ busy time was just in the current * window then that is all that need be accounted. */ if (mark_start > window_start) { - *curr_runnable_sum = scale_exec_time(irqtime, rq); + *curr_runnable_sum += scale_exec_time(irqtime, rq); return; } @@ -1776,7 +1783,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; - wrq->curr_runnable_sum = scale_exec_time(delta, rq); + wrq->curr_runnable_sum += scale_exec_time(delta, rq); return; } @@ -3187,7 +3194,6 @@ static void transfer_busy_time(struct rq *rq, wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); new_task = is_new_task(p); @@ -4088,7 +4094,9 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) rq_lock_irqsave(rq, &rf); old_load = task_load(p); wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + + if (is_idle_task(rq->curr) && p->in_iowait) + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); note_task_waking(p, wallclock); rq_unlock_irqrestore(rq, &rf);