github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-08-02 05:12:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/walt: walt irq work reduce locking

Browse Source 
Optimize walt_irq_work to reduce locking for migrations.

Reduce scheduler overheads by reducing the locks needed
for the walt irq work migrations. Precisely track the
cpus locked, and unlock them at conclusion of work function.

Change-Id: I6ea6d24dc69e9876387253b489e91f82e45ed6d0
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>
This commit is contained in:
Stephen Dickey 2022-01-06 14:59:09 -08:00 committed by Rishabh Bhatnagar
parent ffce2634bc
commit 66f7fd76df
1 changed files with 96 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
kernel/sched/walt/walt.c
View File

@ -2143,7 +2143,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
wts->cpu_cycles = cur_cycles;
}
static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq)
static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq)
{
u64 result;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
@ -2194,7 +2194,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even
done:
wts->mark_start = wallclock;
run_walt_irq_work(old_window_start, rq);
run_walt_irq_work_rollover(old_window_start, rq);
}
static inline void __sched_fork_init(struct task_struct *p)
@ -3372,48 +3372,49 @@ static void walt_update_irqload(struct rq *rq)
wrq->high_irqload = false;
}
/*
* Runs in hard-irq context. This should ideally run just after the latest
* window roll-over.
/**
* __walt_irq_work_locked() - common function to process work
* @is_migration: if true, performing migration work, else rollover
* @lock_cpus: mask of the cpus involved in the operation.
*
* In rq locked context, update the cluster group load and find
* the load of the min cluster, while tracking the total aggregate
* work load. Update the cpufreq through the walt governor,
* based upon the new load calculated.
*
* For the window rollover case lock_cpus will be all possible cpus,
* and for migrations it will include the cpus from the two clusters
* involved in the migration.
*/
static void walt_irq_work(struct irq_work *irq_work)
static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *lock_cpus)
{
struct walt_sched_cluster *cluster;
struct rq *rq;
int cpu;
u64 wc;
bool is_migration = false, is_asym_migration = false;
bool is_asym_migration = false;
u64 total_grp_load = 0, min_cluster_grp_load = 0;
int level = 0;
unsigned long flags;
struct walt_rq *wrq;
/* Am I the window rollover work or the migration work? */
if (irq_work == &walt_migration_irq_work)
is_migration = true;
for_each_cpu(cpu, cpu_possible_mask) {
if (level == 0)
raw_spin_lock(&cpu_rq(cpu)->__lock);
else
raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level);
level++;
}
wc = walt_ktime_get_ns();
walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws);
for_each_sched_cluster(cluster) {
u64 aggr_grp_load = 0;
raw_spin_lock(&cluster->load_lock);
for_each_cpu(cpu, &cluster->cpus) {
rq = cpu_rq(cpu);
wrq = (struct walt_rq *) rq->android_vendor_data1;
if (rq->curr) {
walt_update_task_ravg(rq->curr, rq,
TASK_UPDATE, wc, 0);
account_load_subtractions(rq);
/* only update ravg for locked cpus */
if (cpumask_intersects(lock_cpus, &cluster->cpus)) {
walt_update_task_ravg(rq->curr, rq,
TASK_UPDATE, wc, 0);
account_load_subtractions(rq);
}
/* update aggr_grp_load for all clusters, all cpus */
aggr_grp_load +=
wrq->grp_time.prev_runnable_sum;
}
@ -3423,13 +3424,13 @@ static void walt_irq_work(struct irq_work *irq_work)
wrq->notif_pending = false;
}
}
raw_spin_unlock(&cluster->load_lock);
cluster->aggr_grp_load = aggr_grp_load;
total_grp_load += aggr_grp_load;
if (is_min_capacity_cluster(cluster))
min_cluster_grp_load = aggr_grp_load;
raw_spin_unlock(&cluster->load_lock);
}
if (total_grp_load) {
@ -3453,6 +3454,10 @@ static void walt_irq_work(struct irq_work *irq_work)
cpumask_t cluster_online_cpus;
unsigned int num_cpus, i = 1;
/* for migration, skip unnotified clusters */
if (is_migration && !cpumask_intersects(lock_cpus, &cluster->cpus))
continue;
cpumask_and(&cluster_online_cpus, &cluster->cpus,
cpu_online_mask);
num_cpus = cpumask_weight(&cluster_online_cpus);
@ -3514,8 +3519,73 @@ static void walt_irq_work(struct irq_work *irq_work)
}
spin_unlock_irqrestore(&sched_ravg_window_lock, flags);
}
}
for_each_cpu(cpu, cpu_possible_mask)
/**
* irq_work_restrict_to_mig_clusters() - only allow notified clusters
* @lock_cpus: mask of the cpus for which the runque should be locked.
*
* Remove cpus in clusters that are not part of the migration, using
* the notif_pending flag to track.
*
* This is only valid for the migration irq work.
*/
static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus)
{
struct walt_sched_cluster *cluster;
struct rq *rq;
struct walt_rq *wrq;
int cpu;
for_each_sched_cluster(cluster) {
for_each_cpu(cpu, &cluster->cpus) {
rq = cpu_rq(cpu);
wrq = (struct walt_rq *)rq->android_vendor_data1;
/* remove this cluster if it's not being notified */
if (!wrq->notif_pending) {
cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus);
break;
}
}
}
}
/**
* walt_irq_work() - perform walt irq work for rollover and migration
*
* Process a workqueue call scheduled, while running in a hard irq
* protected context. Handle migration and window rollover work
* with common funtionality, and on window rollover ask core control
* to decide if it needs to adjust the active cpus.
*/
static void walt_irq_work(struct irq_work *irq_work)
{
cpumask_t lock_cpus;
struct walt_rq *wrq;
int level = 0;
int cpu;
bool is_migration = false;
if (irq_work == &walt_migration_irq_work)
is_migration = true;
cpumask_copy(&lock_cpus, cpu_possible_mask);
if (is_migration)
irq_work_restrict_to_mig_clusters(&lock_cpus);
for_each_cpu(cpu, &lock_cpus) {
if (level == 0)
raw_spin_lock(&cpu_rq(cpu)->__lock);
else
raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level);
level++;
}
__walt_irq_work_locked(is_migration, &lock_cpus);
for_each_cpu(cpu, &lock_cpus)
raw_spin_unlock(&cpu_rq(cpu)->__lock);
if (!is_migration) {