From 4240156af1af6aadaa6fd569ccd5246b256e3cc7 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 7 Mar 2022 12:16:43 -0800 Subject: [PATCH] sched/walt/core_ctl: do core control immediately The current code requires an execution of the core control kthread in order to perform any pause/unpause operations. Because the mechanism to pause or unpause a cpu (halt) is only a bit-flip operation, this can cause unnecessary delay in the unpause path. Use spinlocks in halt, and call the main core control functionality immediately from the walt_irq_work context, eliminating this potential delay. Change-Id: I74d82c445097c1074ad106501b101a06875da38e Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 3 ++- kernel/sched/walt/walt_halt.c | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 52e78df1cd2d..6cee4ad08efd 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -85,6 +85,7 @@ ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); static unsigned int last_nr_big; static unsigned int get_active_cpu_count(const struct cluster_data *cluster); +static void __ref do_core_ctl(void); /* ========================= sysfs interface =========================== */ @@ -995,7 +996,7 @@ void core_ctl_check(u64 window_start) wakeup |= eval_need(cluster); if (wakeup) - wake_up_core_ctl_thread(); + do_core_ctl(); core_ctl_call_notifier(); } diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index c622115adae4..abf9b328c9b6 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -15,7 +15,8 @@ /* if a cpu is halting */ struct cpumask __cpu_halt_mask; -static DEFINE_MUTEX(halt_lock); +/* spin lock to allow calling from non-preemptible context */ +static DEFINE_RAW_SPINLOCK(halt_lock); struct halt_cpu_state { u64 last_halt; @@ -359,8 +360,9 @@ int walt_halt_cpus(struct cpumask *cpus) { int ret = 0; cpumask_t requested_cpus; + unsigned long flags; - mutex_lock(&halt_lock); + raw_spin_lock_irqsave(&halt_lock, flags); cpumask_copy(&requested_cpus, cpus); @@ -380,7 +382,7 @@ int walt_halt_cpus(struct cpumask *cpus) else update_ref_counts(&requested_cpus, true); unlock: - mutex_unlock(&halt_lock); + raw_spin_unlock_irqrestore(&halt_lock, flags); return ret; } @@ -397,8 +399,9 @@ int walt_start_cpus(struct cpumask *cpus) { int ret = 0; cpumask_t requested_cpus; + unsigned long flags; - mutex_lock(&halt_lock); + raw_spin_lock_irqsave(&halt_lock, flags); cpumask_copy(&requested_cpus, cpus); update_ref_counts(&requested_cpus, false); @@ -414,7 +417,7 @@ int walt_start_cpus(struct cpumask *cpus) update_ref_counts(&requested_cpus, true); } - mutex_unlock(&halt_lock); + raw_spin_unlock_irqrestore(&halt_lock, flags); return ret; }