github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 03:27:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/walt: kick unhalted cpu for load-balancing

Browse Source 
When a cpu is unhalted it has no tasks enqueued on it. This means
that nothing will be immediately scheduled on that cpu.

Kick the cpu that has been unhalted, to ensure it can load-balance
and help other cpus.

Change-Id: I23fd12bd9967d6aabdbd13b868e0dc316f118342
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>
This commit is contained in:
Stephen Dickey 2022-03-08 11:11:40 -08:00 committed by Rishabh Bhatnagar
parent 2201f39e1c
commit 05cfb9a3e5
4 changed files with 34 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
kernel/sched/walt/walt.c
View File

@ -3078,6 +3078,30 @@ static void walt_update_tg_pointer(struct cgroup_subsys_state *css)
walt_init_tg(css_tg(css));
}
void walt_kick_cpu(int cpu)
{
unsigned int flags = NOHZ_KICK_MASK;
if (cpu == -1)
return;
/*
* Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
* the first flag owns it; cleared by nohz_csd_func().
*/
flags = atomic_fetch_or(flags, nohz_flags(cpu));
if (flags & NOHZ_KICK_MASK)
return;
/*
* This way we generate an IPI on the target CPU which
* is idle. And the softirq performing nohz idle load balance
* will be run before returning from the IPI.
*/
smp_call_function_single_async(cpu, &cpu_rq(cpu)->nohz_csd);
}
static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css)
{
if (unlikely(walt_disabled))

1
kernel/sched/walt/walt.h
View File

@ -161,6 +161,7 @@ extern int sched_set_boost(int enable);
extern void walt_boost_init(void);
extern int sched_pause_cpus(struct cpumask *pause_cpus);
extern int sched_unpause_cpus(struct cpumask *unpause_cpus);
extern void walt_kick_cpu(int cpu);
extern unsigned int sched_get_cpu_util_pct(int cpu);
extern void sched_update_hyst_times(void);

9
kernel/sched/walt/walt_halt.c
View File

@ -268,11 +268,16 @@ static int start_cpus(struct cpumask *cpus)
for_each_cpu(cpu, cpus) {
halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
halt_cpu_state->last_halt = 0;
/* guarantee zero'd last_halt before clearing from the mask */
wmb();
/* wmb to guarantee zero'd last_halt before clearing from the mask */
cpumask_clear_cpu(cpu, cpu_halt_mask);
/* kick the cpu so it can pull tasks
* after the mask has been cleared.
*/
walt_kick_cpu(cpu);
}
trace_halt_cpus(cpus, start_time, 0, 0);

27
kernel/sched/walt/walt_lb.c
View File

@ -758,29 +758,6 @@ static bool should_help_min_cap(int this_cpu)
return false;
}
static void kick_first_idle(int first_idle)
{
unsigned int flags = NOHZ_KICK_MASK;
if (first_idle == -1)
return;
/*
* Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
* the first flag owns it; cleared by nohz_csd_func().
*/
flags = atomic_fetch_or(flags, nohz_flags(first_idle));
if (flags & NOHZ_KICK_MASK)
return;
/*
* This way we generate an IPI on the target CPU which
* is idle. And the softirq performing nohz idle load balance
* will be run before returning from the IPI.
*/
smp_call_function_single_async(first_idle, &cpu_rq(first_idle)->nohz_csd);
}
/* similar to sysctl_sched_migration_cost */
#define NEWIDLE_BALANCE_THRESHOLD 500000
static void walt_newidle_balance(void *unused, struct rq *this_rq,
@ -866,7 +843,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
kick_first_idle(first_idle);
walt_kick_cpu(first_idle);
}
} else if (order_index == 2) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
@ -887,7 +864,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
kick_first_idle(first_idle);
walt_kick_cpu(first_idle);
}
} else {
busy_cpu =