diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2496935cc2fe..8a57b2c96f52 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -342,6 +342,9 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, pred_demand_delta); } +static void rollover_cpu_window(struct rq *rq, bool full_window); +static void rollover_top_tasks(struct rq *rq, bool full_window); + /* * Demand aggregation for frequency purpose: * @@ -394,6 +397,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) int nr_windows; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; u64 old_window_start = wrq->window_start; + bool full_window; if (wallclock < wrq->latest_clock) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than latest_clock=%llu(0x%llx)", @@ -417,6 +421,10 @@ update_window_start(struct rq *rq, u64 wallclock, int event) wrq->prev_window_size = sched_ravg_window; + full_window = nr_windows > 1; + rollover_cpu_window(rq, full_window); + rollover_top_tasks(rq, full_window); + return old_window_start; } @@ -1010,14 +1018,20 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); - walt_update_task_ravg(task_rq(p)->curr, task_rq(p), - TASK_UPDATE, - wallclock, 0); - walt_update_task_ravg(dest_rq->curr, dest_rq, - TASK_UPDATE, wallclock, 0); - - walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, - wallclock, 0); + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); + /* + * The above update might have rolled over the + * window for this migrating task. Since we are + * going to adjust the destination CPU's busy time + * counters with the task busytime counters, roll over + * the window for the destination CPU also. + * + * The update_window_start() does nothing if the window + * is not rolled over, so there is no need to check for + * window boundary or if the counters will be accessed + * or not. + */ + update_window_start(dest_rq, wallclock, TASK_UPDATE); update_task_cpu_cycles(p, new_cpu, wallclock); @@ -1586,11 +1600,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, new_task = is_new_task(p); - if (p_is_curr_task && new_window) { - rollover_cpu_window(rq, full_window); - rollover_top_tasks(rq, full_window); - } - if (!account_busy_for_cpu_time(rq, p, irqtime, event)) goto done; @@ -1631,15 +1640,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, goto done; } + /* + * situations below this need window rollover, which should already be done + * in update_window_start() + */ + if (!p_is_curr_task) { /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has also started, but p is not the current task, so the - * window is not rolled over - just split up and account - * as necessary into curr and prev. The window is only - * rolled over when a new window is processed for the current - * task. + * must have been started in udpate_window_start() + * - just split up and account as necessary into curr and prev. * * Irqtime can't be accounted by a task that isn't the * currently running task. @@ -1684,8 +1695,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. If any of these three above conditions are true + * must have been started in udpate_window_start() + * If any of these three above conditions are true * then this busy time can't be accounted as irqtime. * * Busy time for the idle task need not be accounted. @@ -1717,10 +1728,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, } } - /* - * Rollover is done here by overwriting the values in - * prev_runnable_sum and curr_runnable_sum. - */ *prev_runnable_sum += delta; if (new_task) *nt_prev_runnable_sum += delta; @@ -1743,8 +1750,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. The current task must be the idle task because + * must have been started in udpate_window_start() + * The current task must be the idle task because * irqtime is not accounted for any other task. * * Irqtime will be accounted each time we process IRQ activity @@ -1756,11 +1763,11 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, mark_start = wallclock - irqtime; /* - * Roll window over. If IRQ busy time was just in the current + * If IRQ busy time was just in the current * window then that is all that need be accounted. */ if (mark_start > window_start) { - *curr_runnable_sum = scale_exec_time(irqtime, rq); + *curr_runnable_sum += scale_exec_time(irqtime, rq); return; } @@ -1776,7 +1783,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; - wrq->curr_runnable_sum = scale_exec_time(delta, rq); + wrq->curr_runnable_sum += scale_exec_time(delta, rq); return; } @@ -3187,7 +3194,6 @@ static void transfer_busy_time(struct rq *rq, wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); new_task = is_new_task(p); @@ -4088,7 +4094,9 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) rq_lock_irqsave(rq, &rf); old_load = task_load(p); wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + + if (is_idle_task(rq->curr) && p->in_iowait) + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); note_task_waking(p, wallclock); rq_unlock_irqrestore(rq, &rf);