From 05cfb9a3e5ab6ca306407fc80974c4b3e1a95de3 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 8 Mar 2022 11:11:40 -0800 Subject: [PATCH] sched/walt: kick unhalted cpu for load-balancing When a cpu is unhalted it has no tasks enqueued on it. This means that nothing will be immediately scheduled on that cpu. Kick the cpu that has been unhalted, to ensure it can load-balance and help other cpus. Change-Id: I23fd12bd9967d6aabdbd13b868e0dc316f118342 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 24 ++++++++++++++++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_halt.c | 9 +++++++-- kernel/sched/walt/walt_lb.c | 27 ++------------------------- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8ce20759a5d6..09bf4f5ba25c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3078,6 +3078,30 @@ static void walt_update_tg_pointer(struct cgroup_subsys_state *css) walt_init_tg(css_tg(css)); } +void walt_kick_cpu(int cpu) +{ + unsigned int flags = NOHZ_KICK_MASK; + + if (cpu == -1) + return; + + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ + flags = atomic_fetch_or(flags, nohz_flags(cpu)); + if (flags & NOHZ_KICK_MASK) + return; + + /* + * This way we generate an IPI on the target CPU which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_call_function_single_async(cpu, &cpu_rq(cpu)->nohz_csd); +} + + static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css) { if (unlikely(walt_disabled)) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5a973f5b12c7..4a0ade0c0d60 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -161,6 +161,7 @@ extern int sched_set_boost(int enable); extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); +extern void walt_kick_cpu(int cpu); extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 8e4925f7b439..8e89a82d26c3 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -268,11 +268,16 @@ static int start_cpus(struct cpumask *cpus) for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); halt_cpu_state->last_halt = 0; - - /* guarantee zero'd last_halt before clearing from the mask */ wmb(); + /* wmb to guarantee zero'd last_halt before clearing from the mask */ + cpumask_clear_cpu(cpu, cpu_halt_mask); + + /* kick the cpu so it can pull tasks + * after the mask has been cleared. + */ + walt_kick_cpu(cpu); } trace_halt_cpus(cpus, start_time, 0, 0); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e8d182d90f18..ecc873d69a30 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -758,29 +758,6 @@ static bool should_help_min_cap(int this_cpu) return false; } -static void kick_first_idle(int first_idle) -{ - unsigned int flags = NOHZ_KICK_MASK; - - if (first_idle == -1) - return; - - /* - * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets - * the first flag owns it; cleared by nohz_csd_func(). - */ - flags = atomic_fetch_or(flags, nohz_flags(first_idle)); - if (flags & NOHZ_KICK_MASK) - return; - - /* - * This way we generate an IPI on the target CPU which - * is idle. And the softirq performing nohz idle load balance - * will be run before returning from the IPI. - */ - smp_call_function_single_async(first_idle, &cpu_rq(first_idle)->nohz_csd); -} - /* similar to sysctl_sched_migration_cost */ #define NEWIDLE_BALANCE_THRESHOLD 500000 static void walt_newidle_balance(void *unused, struct rq *this_rq, @@ -866,7 +843,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], @@ -887,7 +864,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else { busy_cpu =