From 86a2dd10daab1b9e74894649b15460ee3f668344 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 5 May 2022 11:08:38 -0700 Subject: [PATCH] sched/walt: Kick newidle balance after unhalting When a cpu is unhalted currently, it will attempt to perform a balance through the normal ipi kick interface. However, this can fail because a balance might have happened in the current 4mS tick, while the cpu was still halted. The next kick as a result of unhalting, fails to execute (min 1 tick between kicks). Instead, use walt_newidle_balance() to ensure that this cpu can take a task. Perform an smp async function call to the target cpu, invoke newidle balance, and allow the cpu to pull tasks immediately after unhalting. Change-Id: Ide1bb51f80dc8c4eba079588ffb6a7ed341e8628 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_halt.c | 2 +- kernel/sched/walt/walt_lb.c | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 47a4168cb00e..08d958eacaf0 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -163,6 +163,7 @@ extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); extern void walt_kick_cpu(int cpu); +extern void walt_smp_call_newidle_balance(int cpu); extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 4a0c8f35f0af..346180098231 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -325,7 +325,7 @@ static int start_cpus(struct cpumask *cpus) /* kick the cpu so it can pull tasks * after the mask has been cleared. */ - walt_kick_cpu(cpu); + walt_smp_call_newidle_balance(cpu); } trace_halt_cpus(cpus, start_time, 0, 0); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 212247934c17..f6ccf195eec2 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -918,6 +918,34 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, help_min_cap, enough_idle); } +/* run newidle balance as a result of an unhalt operation */ +void walt_smp_newidle_balance(void *ignored) +{ + int cpu = raw_smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + int pulled_task; + int done = 0; + + rq_lock(rq, &rf); + update_rq_clock(rq); + walt_newidle_balance(NULL, rq, &rf, &pulled_task, &done); + resched_curr(rq); + rq_unlock(rq, &rf); +} + +static DEFINE_PER_CPU(call_single_data_t, nib_csd); + +void walt_smp_call_newidle_balance(int cpu) +{ + call_single_data_t *csd = &per_cpu(nib_csd, cpu); + + if (unlikely(walt_disabled)) + return; + + smp_call_function_single_async(cpu, csd); +} + static void walt_find_busiest_queue(void *unused, int dst_cpu, struct sched_group *group, struct cpumask *env_cpus, @@ -1028,6 +1056,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, void walt_lb_init(void) { + int cpu; + walt_lb_rotate_work_init(); register_trace_android_rvh_migrate_queued_task(walt_migrate_queued_task, NULL); @@ -1035,4 +1065,11 @@ void walt_lb_init(void) register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); + + for_each_cpu(cpu, cpu_possible_mask) { + call_single_data_t *csd; + + csd = &per_cpu(nib_csd, cpu); + INIT_CSD(csd, walt_smp_newidle_balance, (void *)(unsigned long)cpu); + } }