From f7e7a52285e35421dd02676e59f9ef018f449146 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 18 Oct 2022 21:00:05 -0700 Subject: [PATCH] sched/walt: Change to comparing capacities instead of clusters We want to be able to make load balancing and placement decisions based on the capacities of CPUs as opposed to which cluster they belong to. This method is necessary when dealing with topologies which have multiple clusters for same capacity CPUs. Change-Id: Iff48a59bfc6f3a054360f5ee84347a1fd30447fb Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.h | 9 ++++++--- kernel/sched/walt/walt_cfs.c | 15 ++++----------- kernel/sched/walt/walt_lb.c | 31 +++++++++++-------------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1ded08bab0a2..c72e68120ee5 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -764,6 +764,11 @@ static inline bool task_in_related_thread_group(struct task_struct *p) return (rcu_access_pointer(wts->grp) != NULL); } +static bool check_for_higher_capacity(int cpu1, int cpu2) +{ + return capacity_orig_of(cpu1) > capacity_orig_of(cpu2); +} + extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; static inline bool task_fits_capacity(struct task_struct *p, @@ -771,13 +776,11 @@ static inline bool task_fits_capacity(struct task_struct *p, int cpu) { unsigned int margin; - struct walt_rq *src_wrq = &per_cpu(walt_rq, task_cpu(p)); - struct walt_rq *dst_wrq = &per_cpu(walt_rq, cpu); /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. */ - if (src_wrq->cluster->id > dst_wrq->cluster->id) { + if (check_for_higher_capacity(task_cpu(p), cpu)) { margin = sched_capacity_margin_down[cpu]; if (task_in_related_thread_group(p)) { if (is_min_cluster_cpu(cpu)) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index b008c455b09a..456f5560743b 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -67,9 +67,7 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) bool base_test = cpumask_test_cpu(cpu, p->cpus_ptr) && cpu_active(cpu); - struct walt_rq *wrq = &per_cpu(walt_rq, cpu); - struct walt_rq *start_wrq = &per_cpu(walt_rq, start_cpu); - bool start_cap_test = (wrq->cluster->id >= start_wrq->cluster->id); + bool start_cap_test = !check_for_higher_capacity(start_cpu, cpu); return base_test && start_cap_test; } @@ -748,12 +746,10 @@ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) { bool new_cpu_is_idle = available_idle_cpu(cpu); bool best_cpu_is_idle = available_idle_cpu(best_cpu); - struct walt_rq *wrq = &per_cpu(walt_rq, cpu); - struct walt_rq *best_wrq = &per_cpu(walt_rq, best_cpu); - if (best_wrq->cluster->id < wrq->cluster->id) + if (check_for_higher_capacity(cpu, best_cpu)) return false; - if (wrq->cluster->id < best_wrq->cluster->id) + if (check_for_higher_capacity(best_cpu, cpu)) return true; if (best_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) @@ -806,8 +802,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int first_cpu; bool energy_eval_needed = true; struct compute_energy_output output; - struct walt_rq *prev_wrq = &per_cpu(walt_rq, prev_cpu); - struct walt_rq *start_wrq; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && cpumask_test_cpu(prev_cpu, p->cpus_ptr)) @@ -819,7 +813,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, &energy_eval_needed); start_cpu = cpumask_first(&cpu_array[order_index][0]); - start_wrq = &per_cpu(walt_rq, start_cpu); is_rtg = task_in_related_thread_group(p); curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); @@ -953,7 +946,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && (prev_energy != ULONG_MAX) && (best_energy_cpu != prev_cpu) && ((prev_energy - best_energy) <= prev_energy >> 5) && - (prev_wrq->cluster->id <= start_wrq->cluster->id)) + !check_for_higher_capacity(prev_cpu, start_cpu)) best_energy_cpu = prev_cpu; unlock: diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 02c14e6bbda7..be8027a21d54 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -271,13 +271,11 @@ static inline bool need_active_lb(struct task_struct *p, int dst_cpu, int src_cpu) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); - struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); if (cpu_rq(src_cpu)->active_balance) return false; - if (dst_wrq->cluster->id <= src_wrq->cluster->id) + if (!check_for_higher_capacity(dst_cpu, src_cpu)) return false; if (!wts->misfit) @@ -294,15 +292,14 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) struct task_struct *pulled_task = NULL, *p; bool active_balance = false, to_lower, to_higher; struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); - struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); struct walt_task_struct *wts; struct task_struct *pull_me; int task_visited; BUG_ON(src_cpu == dst_cpu); - to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; - to_higher = dst_wrq->cluster->id > src_wrq->cluster->id; + to_lower = check_for_higher_capacity(src_cpu, dst_cpu); + to_higher = check_for_higher_capacity(dst_cpu, src_cpu); raw_spin_lock_irqsave(&src_rq->__lock, flags); @@ -620,18 +617,16 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; - struct walt_rq *fsrc_wrq = &per_cpu(walt_rq, fsrc_cpu); - struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); - if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) - busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, - src_mask, has_misfit, is_newidle); - else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id) + if (check_for_higher_capacity(dst_cpu, fsrc_cpu)) busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, src_mask, has_misfit, is_newidle); - else + else if (check_for_higher_capacity(fsrc_cpu, dst_cpu)) busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, src_mask, has_misfit, is_newidle); + else + busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, + src_mask, has_misfit, is_newidle); return busiest_cpu; } @@ -643,7 +638,6 @@ void walt_lb_tick(struct rq *rq) struct task_struct *p = rq->curr; unsigned long flags; struct walt_rq *prev_wrq = &per_cpu(walt_rq, cpu_of(rq)); - struct walt_rq *new_wrq; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; raw_spin_lock(&rq->__lock); @@ -673,10 +667,9 @@ void walt_lb_tick(struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); - new_wrq = &per_cpu(walt_rq, new_cpu); /* prevent active task migration to busy or same/lower capacity CPU */ - if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id) + if (!available_idle_cpu(new_cpu) || !check_for_higher_capacity(new_cpu, prev_cpu)) goto out_unlock; raw_spin_lock(&rq->__lock); @@ -1088,13 +1081,11 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, int dst_cpu, int *can_migrate) { bool to_lower, to_higher; - struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); - struct walt_rq *task_wrq = &per_cpu(walt_rq, task_cpu(p)); if (unlikely(walt_disabled)) return; - to_lower = dst_wrq->cluster->id < task_wrq->cluster->id; - to_higher = dst_wrq->cluster->id > task_wrq->cluster->id; + to_lower = check_for_higher_capacity(task_cpu(p), dst_cpu); + to_higher = check_for_higher_capacity(dst_cpu, task_cpu(p)); if (_walt_can_migrate_task(p, dst_cpu, to_lower, to_higher, true))