diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8ce20759a5d6..09bf4f5ba25c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3078,6 +3078,30 @@ static void walt_update_tg_pointer(struct cgroup_subsys_state *css) walt_init_tg(css_tg(css)); } +void walt_kick_cpu(int cpu) +{ + unsigned int flags = NOHZ_KICK_MASK; + + if (cpu == -1) + return; + + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ + flags = atomic_fetch_or(flags, nohz_flags(cpu)); + if (flags & NOHZ_KICK_MASK) + return; + + /* + * This way we generate an IPI on the target CPU which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_call_function_single_async(cpu, &cpu_rq(cpu)->nohz_csd); +} + + static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css) { if (unlikely(walt_disabled)) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5a973f5b12c7..4a0ade0c0d60 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -161,6 +161,7 @@ extern int sched_set_boost(int enable); extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); +extern void walt_kick_cpu(int cpu); extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 8e4925f7b439..8e89a82d26c3 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -268,11 +268,16 @@ static int start_cpus(struct cpumask *cpus) for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); halt_cpu_state->last_halt = 0; - - /* guarantee zero'd last_halt before clearing from the mask */ wmb(); + /* wmb to guarantee zero'd last_halt before clearing from the mask */ + cpumask_clear_cpu(cpu, cpu_halt_mask); + + /* kick the cpu so it can pull tasks + * after the mask has been cleared. + */ + walt_kick_cpu(cpu); } trace_halt_cpus(cpus, start_time, 0, 0); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e8d182d90f18..ecc873d69a30 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -758,29 +758,6 @@ static bool should_help_min_cap(int this_cpu) return false; } -static void kick_first_idle(int first_idle) -{ - unsigned int flags = NOHZ_KICK_MASK; - - if (first_idle == -1) - return; - - /* - * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets - * the first flag owns it; cleared by nohz_csd_func(). - */ - flags = atomic_fetch_or(flags, nohz_flags(first_idle)); - if (flags & NOHZ_KICK_MASK) - return; - - /* - * This way we generate an IPI on the target CPU which - * is idle. And the softirq performing nohz idle load balance - * will be run before returning from the IPI. - */ - smp_call_function_single_async(first_idle, &cpu_rq(first_idle)->nohz_csd); -} - /* similar to sysctl_sched_migration_cost */ #define NEWIDLE_BALANCE_THRESHOLD 500000 static void walt_newidle_balance(void *unused, struct rq *this_rq, @@ -866,7 +843,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], @@ -887,7 +864,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else { busy_cpu =