From 04c68623ec61cfb831513455f08f1eb7a5ad47a5 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 07:13:20 +0530 Subject: [PATCH] sched/walt: Simplify core_ctl active cpu count management The core_ctl has to know how many CPUs are active in each cluster. If the current need is less than the number of active CPUs, it tries to pause the CPUs provided tunables are in agreement. On the other hand, if the current need is more than the number of active CPUs, it tries to resume the CPUs, provided those CPUs are paused by us in the first place. To determine the number of active CPUs, we should consider only active CPUs. The others could be in pause or hotplugged out. Since we only use cpu_active_mask for both, we don't need wrappers to tell this to us. All we need is to maintain how many CPUs that are going to be paused/resumed in the current iteration and check against the need/limits. Change-Id: I0a29da869b52eff35ff1dfa3094c51d6ece4e49f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/core_ctl.c | 87 +++++++++++++++--------------------- kernel/sched/walt/qc_vas.c | 52 --------------------- kernel/sched/walt/walt.h | 3 -- 4 files changed, 37 insertions(+), 107 deletions(-) delete mode 100644 kernel/sched/walt/qc_vas.c diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index 34987fd075d2..18e5264cf702 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o qc_vas.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index f02888e95d3f..c142a9ad8ad3 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -752,8 +752,10 @@ static unsigned int apply_limits(const struct cluster_data *cluster, static unsigned int get_active_cpu_count(const struct cluster_data *cluster) { - return cluster->num_cpus - - sched_pause_count(&cluster->cpu_mask, true); + cpumask_t cpus; + + cpumask_and(&cpus, &cluster->cpu_mask, cpu_active_mask); + return cpumask_weight(&cpus); } static bool is_active(const struct cpu_data *state) @@ -981,14 +983,11 @@ void core_ctl_check(u64 window_start) core_ctl_call_notifier(); } +/* must be called with state_lock held */ static void move_cpu_lru(struct cpu_data *cpu_data) { - unsigned long flags; - - spin_lock_irqsave(&state_lock, flags); list_del(&cpu_data->sib); list_add_tail(&cpu_data->sib, &cpu_data->cluster->lru); - spin_unlock_irqrestore(&state_lock, flags); } static void try_to_pause(struct cluster_data *cluster, unsigned int need, @@ -997,7 +996,7 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, struct cpu_data *c, *tmp; unsigned long flags; unsigned int num_cpus = cluster->num_cpus; - unsigned int nr_paused = 0; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; bool first_pass = cluster->nr_not_preferred_cpus; /* @@ -1011,7 +1010,7 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (!is_active(c)) continue; - if (cluster->active_cpus == need) + if (active_cpus - nr_pending == need) break; /* Don't pause busy CPUs. */ if (c->is_busy) @@ -1024,75 +1023,57 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (cluster->nr_not_preferred_cpus && !c->not_preferred) continue; - spin_unlock_irqrestore(&state_lock, flags); - pr_debug("Trying to pause CPU%u\n", c->cpu); - cpumask_set_cpu(c->cpu, pause_cpus); - sched_pause_pending(c->cpu); - + nr_pending++; c->paused_by_us = true; + cluster->nr_paused_cpus++; move_cpu_lru(c); - nr_paused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus += nr_paused; - spin_unlock_irqrestore(&state_lock, flags); again: /* * If the number of active CPUs is within the limits, then * don't force pause of any busy CPUs. */ - if (cluster->active_cpus <= cluster->max_cpus) - return; + if (active_cpus - nr_pending <= cluster->max_cpus) + goto unlock; - nr_paused = 0; num_cpus = cluster->num_cpus; - spin_lock_irqsave(&state_lock, flags); list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { if (!num_cpus--) break; if (!is_active(c)) continue; - if (cluster->active_cpus <= cluster->max_cpus) + if (active_cpus - nr_pending <= cluster->max_cpus) break; if (first_pass && !c->not_preferred) continue; - spin_unlock_irqrestore(&state_lock, flags); - cpumask_set_cpu(c->cpu, pause_cpus); - sched_pause_pending(c->cpu); - + nr_pending++; c->paused_by_us = true; + cluster->nr_paused_cpus++; move_cpu_lru(c); - nr_paused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus += nr_paused; - spin_unlock_irqrestore(&state_lock, flags); - - if (first_pass && cluster->active_cpus > cluster->max_cpus) { + if (first_pass && active_cpus - nr_pending > cluster->max_cpus) { first_pass = false; goto again; } +unlock: + spin_unlock_irqrestore(&state_lock, flags); } -static void __try_to_resume(struct cluster_data *cluster, - unsigned int need, bool force, struct cpumask *unpause_cpus) +static int __try_to_resume(struct cluster_data *cluster, unsigned int need, + bool force, struct cpumask *unpause_cpus) { struct cpu_data *c, *tmp; unsigned long flags; unsigned int num_cpus = cluster->num_cpus; - unsigned int nr_unpaused = 0; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; /* * Protect against entry being removed (and added at tail) by other @@ -1108,35 +1089,38 @@ static void __try_to_resume(struct cluster_data *cluster, if ((cpu_online(c->cpu) && cpu_active(c->cpu)) || (!force && c->not_preferred)) continue; - if (cluster->active_cpus == need) + if (active_cpus + nr_pending == need) break; - spin_unlock_irqrestore(&state_lock, flags); - pr_debug("Trying to resume CPU%u\n", c->cpu); cpumask_set_cpu(c->cpu, unpause_cpus); - sched_unpause_pending(c->cpu); - + nr_pending++; c->paused_by_us = false; + cluster->nr_paused_cpus--; move_cpu_lru(c); - nr_unpaused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus -= nr_unpaused; + spin_unlock_irqrestore(&state_lock, flags); + + return nr_pending; } static void try_to_resume(struct cluster_data *cluster, unsigned int need, struct cpumask *unpause_cpus) { bool force_use_non_preferred = false; + unsigned int nr_pending; - __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); + /* + * __try_to_resume() marks the CPUs to be resumed but active_cpus + * won't be reflected yet. So use the nr_pending to adjust active + * count. + */ + nr_pending = __try_to_resume(cluster, need, force_use_non_preferred, + unpause_cpus); - if (cluster->active_cpus == need) + if (cluster->active_cpus + nr_pending == need) return; force_use_non_preferred = true; @@ -1153,6 +1137,7 @@ static void __ref do_core_ctl(void) for_each_cluster(cluster, index) { + cluster->active_cpus = get_active_cpu_count(cluster); need = apply_limits(cluster, cluster->need_cpus); if (adjustment_possible(cluster, need)) { diff --git a/kernel/sched/walt/qc_vas.c b/kernel/sched/walt/qc_vas.c deleted file mode 100644 index 9db134f8363f..000000000000 --- a/kernel/sched/walt/qc_vas.c +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - */ - -#include -#include -#include -#include - -#include "walt.h" - -#ifdef CONFIG_HOTPLUG_CPU - -cpumask_t pending_active_mask = CPU_MASK_NONE; -int sched_pause_count(const cpumask_t *mask, bool include_offline) -{ - cpumask_t count_mask = CPU_MASK_NONE; - cpumask_t pause_mask = CPU_MASK_NONE; - - if (cpumask_any(&pending_active_mask) >= nr_cpu_ids) { - /* initialize pending_active_state */ - cpumask_copy(&pending_active_mask, cpu_active_mask); - } - - if (include_offline) { - - /* get all offline or paused cpus */ - cpumask_complement(&pause_mask, &pending_active_mask); - cpumask_complement(&count_mask, cpu_online_mask); - cpumask_or(&count_mask, &count_mask, &pause_mask); - - /* get all offline or paused cpus in this cluster */ - cpumask_and(&count_mask, &count_mask, mask); - } else { - cpumask_andnot(&count_mask, mask, &pending_active_mask); - } - - return cpumask_weight(&count_mask); -} - -void sched_pause_pending(int cpu) -{ - cpumask_clear_cpu(cpu, &pending_active_mask); -} - -void sched_unpause_pending(int cpu) -{ - cpumask_set_cpu(cpu, &pending_active_mask); -} - -#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index a423fbf649ca..f1c11dded20c 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -254,9 +254,6 @@ extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); extern u32 sched_get_init_task_load(struct task_struct *p); extern void core_ctl_check(u64 wallclock); extern int sched_set_boost(int enable); -extern int sched_pause_count(const cpumask_t *mask, bool include_offline); -extern void sched_pause_pending(int cpu); -extern void sched_unpause_pending(int cpu); extern int sched_wake_up_idle_show(struct seq_file *m, void *v); extern ssize_t sched_wake_up_idle_write(struct file *file, const char __user *buf, size_t count, loff_t *offset);