github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-30 11:11:26 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge "sched/walt: Early Detect Boost Optimizations"

Browse Source
This commit is contained in:
qctecmdr 2022-09-23 14:30:17 -07:00 committed by Gerrit - the friendly Code Review server
commit d665060fae
8 changed files with 265 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/linux/sched/walt.h
View File

@ -93,6 +93,7 @@ struct walt_task_struct {
* 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions
* 0 = nothing, 1 = enqueued, 2 = dequeued
*/
u32 flags;
u64 mark_start;
u64 window_start;
u32 sum, demand;
@ -141,6 +142,16 @@ struct walt_task_struct {
u8 hung_detect_status;
};
/*
* enumeration to set the flags variable
* each index below represents an offset into
* wts->flags
*/
enum walt_flags {
WALT_INIT,
MAX_WALT_FLAGS
};
#define wts_to_ts(wts) ({ \
void *__mptr = (void *)(wts); \
((struct task_struct *)(__mptr - \

9
kernel/sched/walt/cpufreq_walt.c
View File

@ -296,8 +296,14 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut
bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION;
bool is_rtg_boost = wg_cpu->walt_load.rtgb_active;
bool is_hiload;
bool is_ed_boost = wg_cpu->walt_load.ed_active;
unsigned long pl = wg_cpu->walt_load.pl;
if (is_ed_boost) {
cpu_util = mult_frac(cpu_util, 100 + sysctl_ed_boost_pct, 100);
max_and_reason(util, cpu_util, wg_cpu, CPUFREQ_REASON_EARLY_DET);
}
if (is_rtg_boost)
max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST);
@ -316,6 +322,9 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut
pl = mult_frac(pl, TARGET_LOAD, 100);
max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL);
}
if (is_ed_boost)
wg_cpu->reasons |= CPUFREQ_REASON_EARLY_DET;
}
static inline unsigned long target_util(struct waltgov_policy *wg_policy,

10
kernel/sched/walt/sysctl.c
View File

@ -74,6 +74,7 @@ unsigned int sysctl_sched_asymcap_boost;
unsigned int sysctl_sched_long_running_rt_task_ms;
unsigned int sysctl_sched_idle_enough;
unsigned int sysctl_sched_cluster_util_thres_pct;
unsigned int sysctl_ed_boost_pct;
/* range is [1 .. INT_MAX] */
static int sysctl_task_read_pid = 1;
@ -922,6 +923,15 @@ struct ctl_table walt_table[] = {
.extra1 = SYSCTL_ZERO,
.extra2 = &two_thousand,
},
{
.procname = "sched_ed_boost",
.data = &sysctl_ed_boost_pct,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = &one_hundred,
},
{ }
};

195
kernel/sched/walt/walt.c
View File

@ -354,79 +354,6 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
static void rollover_cpu_window(struct rq *rq, bool full_window);
static void rollover_top_tasks(struct rq *rq, bool full_window);
/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster.
* @start_cpu: The cpu from the cluster to choose from
*
* If the cluster has a 32bit capable cpu return it regardless
* of whether it is halted or not.
*
* If the cluster does not have a 32 bit capable cpu, find the
* first unhalted, active cpu in this cluster.
*/
int walt_find_cluster_packing_cpu(int start_cpu)
{
struct rq *rq = cpu_rq(start_cpu);
struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1;
struct walt_sched_cluster *cluster = wrq->cluster;
cpumask_t unhalted_cpus;
cpumask_t cluster_32bit_cpus;
/* find all 32 bit capable cpus in this cluster */
cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask());
/* pack 32 bit and 64 bit tasks on the same cpu, if possible */
if (cpumask_weight(&cluster_32bit_cpus) > 0)
return cpumask_first(&cluster_32bit_cpus);
/* find all unhalted active cpus */
cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);
/* find all unhalted active cpus in this cluster */
cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);
/* return the first found unhalted, active cpu, in this cluster */
return cpumask_first(&unhalted_cpus);
}
/* for cfs and rt, determine if packing_cpu should be used */
bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p)
{
struct rq *rq;
struct walt_rq *wrq;
struct walt_sched_cluster *cluster;
/* packing cpu must be a valid cpu for runqueue lookup */
if (packing_cpu >= nr_cpu_ids)
return false;
rq = cpu_rq(packing_cpu);
wrq = (struct walt_rq *)rq->android_vendor_data1;
cluster = wrq->cluster;
/* if idle_enough feature is not enabled */
if (!sysctl_sched_idle_enough)
return false;
/* if cpu is not allowed for this task */
if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr))
return false;
/* if cluster util is high */
if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct)
return false;
/* if cpu utilization is high */
if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough)
return false;
/* don't pack big tasks */
if (task_util(p) >= sysctl_sched_idle_enough)
return false;
/* the packing cpu can be used, so pack! */
return true;
}
/*
* Demand aggregation for frequency purpose:
*
@ -657,12 +584,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason)
u64 load, tt_load = 0, kload = 0;
struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq));
if (wrq->ed_task != NULL) {
load = sched_ravg_window;
*reason = CPUFREQ_REASON_EARLY_DET;
goto done;
}
if (sched_freq_aggr_en) {
load = wrq->prev_runnable_sum + aggr_grp_load;
*reason = CPUFREQ_REASON_FREQ_AGR;
@ -694,7 +615,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason)
*reason = CPUFREQ_REASON_SUH;
}
done:
trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en,
load, 0, walt_rotation_enabled,
sysctl_sched_user_hint, wrq, *reason);
@ -732,6 +652,10 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea
walt_load->pl = pl;
walt_load->ws = walt_load_reported_window;
walt_load->rtgb_active = rtgb_active;
if (wrq->ed_task)
walt_load->ed_active = true;
else
walt_load->ed_active = false;
}
return (util >= capacity) ? capacity : util;
@ -2457,6 +2381,7 @@ static void init_new_task_load(struct task_struct *p)
wts->mvp_prio = WALT_NOT_MVP;
wts->cidx = 0;
__sched_fork_init(p);
walt_flag_set(p, WALT_INIT, 1);
}
static void init_existing_task_load(struct task_struct *p)
@ -3798,6 +3723,50 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus)
}
}
static void update_cpu_capacity_helper(int cpu)
{
unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu);
unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu);
unsigned long thermal_cap, old;
struct walt_sched_cluster *cluster;
struct rq *rq = cpu_rq(cpu);
if (unlikely(walt_disabled))
return;
/*
* thermal_pressure = cpu_scale - curr_cap_as_per_thermal.
* so,
* curr_cap_as_per_thermal = cpu_scale - thermal_pressure.
*/
thermal_cap = fmax_capacity - thermal_pressure;
cluster = cpu_cluster(cpu);
/* reduce the fmax_capacity under cpufreq constraints */
if (cluster->max_freq != cluster->max_possible_freq)
fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq,
cluster->max_possible_freq);
old = rq->cpu_capacity_orig;
rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap);
if (old != rq->cpu_capacity_orig)
trace_update_cpu_capacity(cpu, 0, 0);
}
/*
* The intention of this hook is to update cpu_capacity_orig as well as
* (*capacity), otherwise we will end up capacity_of() > capacity_orig_of().
*/
static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity)
{
unsigned long rt_pressure = arch_scale_cpu_capacity(cpu) - *capacity;
update_cpu_capacity_helper(cpu);
*capacity = max((int)(cpu_rq(cpu)->cpu_capacity_orig - rt_pressure), 0);
}
/**
* walt_irq_work() - perform walt irq work for rollover and migration
*
@ -3839,6 +3808,11 @@ static void walt_irq_work(struct irq_work *irq_work)
level++;
}
if (!is_migration) {
for_each_cpu(cpu, &lock_cpus)
update_cpu_capacity_helper(cpu);
}
__walt_irq_work_locked(is_migration, &lock_cpus);
for_each_cpu(cpu, &lock_cpus)
@ -4074,45 +4048,6 @@ static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *polic
cpu_cluster(policy->cpu)->max_freq = policy->max;
}
/*
* The intention of this hook is to update cpu_capacity_orig as well as
* (*capacity), otherwise we will end up capacity_of() > capacity_orig_of().
*/
static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity)
{
unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu);
unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu);
unsigned long thermal_cap, old;
unsigned long rt_pressure = fmax_capacity - *capacity;
struct walt_sched_cluster *cluster;
struct rq *rq = cpu_rq(cpu);
if (unlikely(walt_disabled))
return;
/*
* thermal_pressure = cpu_scale - curr_cap_as_per_thermal.
* so,
* curr_cap_as_per_thermal = cpu_scale - thermal_pressure.
*/
thermal_cap = fmax_capacity - thermal_pressure;
cluster = cpu_cluster(cpu);
/* reduce the fmax_capacity under cpufreq constraints */
if (cluster->max_freq != cluster->max_possible_freq)
fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq,
cluster->max_possible_freq);
old = rq->cpu_capacity_orig;
rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap);
if (old != rq->cpu_capacity_orig)
trace_update_cpu_capacity(cpu, rt_pressure, *capacity);
*capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL);
}
static void android_rvh_sched_cpu_starting(void *unused, int cpu)
{
if (unlikely(walt_disabled))
@ -4255,6 +4190,10 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st
if (!double_enqueue)
walt_inc_cumulative_runnable_avg(rq, p);
if ((flags & ENQUEUE_WAKEUP) && do_pl_notif(rq))
waltgov_run_callback(rq, WALT_CPUFREQ_PL);
trace_sched_enq_deq_task(p, 1, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts));
}
@ -4274,7 +4213,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st
* therefore the check to ensure that prev_on_rq_cpu is needed to prevent
* an invalid failure.
*/
if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq))
if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq) &&
walt_flag_test(p, WALT_INIT))
WALT_BUG(WALT_BUG_UPSTREAM, p, "dequeue cpu %d not same as enqueue %d\n",
cpu_of(rq), wts->prev_on_rq_cpu);
@ -4371,20 +4311,6 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p)
rcu_read_unlock();
}
static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct *p)
{
unsigned long flags;
int cpu = p->cpu;
if (unlikely(walt_disabled))
return;
raw_spin_lock_irqsave(&cpu_rq(cpu)->__lock, flags);
if (do_pl_notif(cpu_rq(cpu)))
waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL);
raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags);
}
static u64 tick_sched_clock;
static DECLARE_COMPLETION(tick_sched_clock_completion);
@ -4556,7 +4482,6 @@ static void register_walt_hooks(void)
register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL);
register_trace_android_rvh_after_dequeue_task(android_rvh_dequeue_task, NULL);
register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL);
register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL);
register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL);
register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL);
register_trace_android_rvh_schedule(android_rvh_schedule, NULL);

89
kernel/sched/walt/walt.h
View File

@ -59,6 +59,7 @@ struct walt_cpu_load {
unsigned long pl;
bool rtgb_active;
u64 ws;
bool ed_active;
};
#define DECLARE_BITMAP_ARRAY(name, nr, bits) \
@ -216,6 +217,7 @@ extern unsigned int sysctl_sched_user_hint;
extern unsigned int sysctl_sched_conservative_pl;
extern unsigned int sysctl_sched_hyst_min_coloc_ns;
extern unsigned int sysctl_sched_long_running_rt_task_ms;
extern unsigned int sysctl_ed_boost_pct;
#define WALT_MANY_WAKEUP_DEFAULT 1000
extern unsigned int sysctl_sched_many_wakeup_threshold;
@ -903,12 +905,31 @@ extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int wr
void __user *buffer, size_t *lenp,
loff_t *ppos);
static inline void walt_flag_set(struct task_struct *p, enum walt_flags feature, bool set)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (set)
wts->flags |= 1 << feature;
else
wts->flags &= ~(1 << feature);
}
static inline bool walt_flag_test(struct task_struct *p, enum walt_flags feature)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
return !!(wts->flags & (1 << feature));
}
#define WALT_MVP_SLICE 3000000U
#define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE)
/* higher number, better priority */
#define WALT_RTG_MVP 0
#define WALT_BINDER_MVP 1
#define WALT_TASK_BOOST_MVP 2
#define WALT_LL_PIPE_MVP 3
#define WALT_NOT_MVP -1
@ -945,6 +966,74 @@ extern struct cpumask __cpu_halt_mask;
#define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask)
#define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask)
/* walt_find_and_choose_cluster_packing_cpu - Return a packing_cpu choice common for this cluster.
* @start_cpu: The cpu from the cluster to choose from
*
* If the cluster has a 32bit capable cpu return it regardless
* of whether it is halted or not.
*
* If the cluster does not have a 32 bit capable cpu, find the
* first unhalted, active cpu in this cluster.
*
* Returns -1 if packing_cpu if not found or is unsuitable to be packed on to
* Returns a valid cpu number if packing_cpu is found and is usable
*/
static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p)
{
struct rq *rq = cpu_rq(start_cpu);
struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1;
struct walt_sched_cluster *cluster = wrq->cluster;
cpumask_t unhalted_cpus;
cpumask_t cluster_32bit_cpus;
int packing_cpu;
/* if idle_enough feature is not enabled */
if (!sysctl_sched_idle_enough)
return -1;
if (!sysctl_sched_cluster_util_thres_pct)
return -1;
/* find all 32 bit capable cpus in this cluster */
cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask());
/* pack 32 bit and 64 bit tasks on the same cpu, if possible */
if (cpumask_weight(&cluster_32bit_cpus) > 0) {
packing_cpu = cpumask_first(&cluster_32bit_cpus);
} else {
/* find all unhalted active cpus */
cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);
/* find all unhalted active cpus in this cluster */
cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);
/* return the first found unhalted, active cpu, in this cluster */
packing_cpu = cpumask_first(&unhalted_cpus);
}
/* packing cpu must be a valid cpu for runqueue lookup */
if (packing_cpu >= nr_cpu_ids)
return -1;
/* if cpu is not allowed for this task */
if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr))
return -1;
/* if cluster util is high */
if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct)
return -1;
/* if cpu utilization is high */
if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough)
return -1;
/* don't pack big tasks */
if (task_util(p) >= sysctl_sched_idle_enough)
return -1;
/* the packing cpu can be used, so pack! */
return packing_cpu;
}
extern void walt_task_dump(struct task_struct *p);
extern void walt_rq_dump(int cpu);
extern void walt_dump(void);

12
kernel/sched/walt/walt_cfs.c
View File

@ -4,6 +4,7 @@
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <linux/seq_file.h>
#include <trace/hooks/sched.h>
#include <trace/hooks/binder.h>
@ -280,8 +281,8 @@ static void walt_find_best_target(struct sched_domain *sd,
}
/* fast path for packing_cpu */
packing_cpu = walt_find_cluster_packing_cpu(start_cpu);
if (walt_choose_packing_cpu(packing_cpu, p)) {
packing_cpu = walt_find_and_choose_cluster_packing_cpu(start_cpu, p);
if (packing_cpu >= 0) {
fbt_env->fastpath = CLUSTER_PACKING_FASTPATH;
cpumask_set_cpu(packing_cpu, candidates);
goto out;
@ -1041,14 +1042,17 @@ static void binder_restore_priority_hook(void *data,
*/
int walt_get_mvp_task_prio(struct task_struct *p)
{
if (walt_procfs_low_latency_task(p) ||
walt_pipeline_low_latency_task(p))
return WALT_LL_PIPE_MVP;
if (per_task_boost(p) == TASK_BOOST_STRICT_MAX)
return WALT_TASK_BOOST_MVP;
if (walt_binder_low_latency_task(p))
return WALT_BINDER_MVP;
if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) ||
walt_pipeline_low_latency_task(p))
if (task_rtg_high_prio(p))
return WALT_RTG_MVP;
return WALT_NOT_MVP;

105
kernel/sched/walt/walt_lb.c
View File

@ -296,17 +296,19 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_task_struct *wts;
struct task_struct *pull_me;
int task_visited;
BUG_ON(src_cpu == dst_cpu);
to_lower = dst_wrq->cluster->id < src_wrq->cluster->id;
to_higher = dst_wrq->cluster->id > src_wrq->cluster->id;
raw_spin_lock_irqsave(&src_rq->__lock, flags);
pull_me = NULL;
task_visited = 0;
list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) {
if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr))
continue;
@ -317,13 +319,30 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
false))
continue;
walt_detach_task(p, src_rq, dst_rq);
pulled_task = p;
if (pull_me == NULL) {
pull_me = p;
} else {
if (to_lower) {
if (task_util(p) < task_util(pull_me))
pull_me = p;
} else if (task_util(p) > task_util(pull_me)) {
pull_me = p;
}
}
task_visited++;
if (task_visited > 5)
break;
}
if (pull_me) {
walt_detach_task(pull_me, src_rq, dst_rq);
pulled_task = pull_me;
goto unlock;
}
pull_me = NULL;
task_visited = 0;
list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) {
if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr))
continue;
@ -334,8 +353,24 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
true))
continue;
walt_detach_task(p, src_rq, dst_rq);
pulled_task = p;
if (pull_me == NULL) {
pull_me = p;
} else {
if (to_lower) {
if (task_util(p) < task_util(pull_me))
pull_me = p;
} else if (task_util(p) > task_util(pull_me)) {
pull_me = p;
}
}
task_visited++;
if (task_visited > 5)
break;
}
if (pull_me) {
walt_detach_task(pull_me, src_rq, dst_rq);
pulled_task = pull_me;
goto unlock;
}
@ -380,7 +415,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
return 0;
raw_spin_lock_irqsave(&dst_rq->__lock, flags);
walt_attach_task(p, dst_rq);
walt_attach_task(pulled_task, dst_rq);
raw_spin_unlock_irqrestore(&dst_rq->__lock, flags);
return 1; /* we pulled 1 task */
@ -426,7 +461,7 @@ static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask)
}
static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask,
int *has_misfit)
int *has_misfit, bool is_newidle)
{
int i;
int busiest_cpu = -1;
@ -452,7 +487,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr
}
static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask,
int *has_misfit)
int *has_misfit, bool is_newidle)
{
int i;
int busiest_cpu = -1;
@ -512,7 +547,7 @@ static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t
}
static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask,
int *has_misfit)
int *has_misfit, bool is_newidle)
{
int i;
int busiest_cpu = -1;
@ -521,6 +556,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t
int total_cpus = 0;
int busy_nr_big_tasks = 0;
struct walt_rq *wrq;
bool treat_dst_idle = is_newidle || available_idle_cpu(dst_cpu);
/*
* A higher capacity CPU is looking at a lower capacity
@ -552,7 +588,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t
/* active migration is allowed only to idle cpu */
if (cpu_rq(i)->cfs.h_nr_running < 2 &&
(!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu)))
(!wrq->walt_stats.nr_big_tasks || !treat_dst_idle))
continue;
if (!walt_rotation_enabled && !cpu_overutilized(i) &&
@ -579,7 +615,8 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t
return busiest_cpu;
}
static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit)
static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit,
bool is_newidle)
{
int fsrc_cpu = cpumask_first(src_mask);
int busiest_cpu;
@ -588,13 +625,13 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int
if (dst_wrq->cluster->id == fsrc_wrq->cluster->id)
busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu,
src_mask, has_misfit);
src_mask, has_misfit, is_newidle);
else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id)
busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu,
src_mask, has_misfit);
src_mask, has_misfit, is_newidle);
else
busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu,
src_mask, has_misfit);
src_mask, has_misfit, is_newidle);
return busiest_cpu;
}
@ -830,7 +867,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
*/
if (num_sched_clusters <= 2) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
@ -839,7 +876,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
find_next_cluster = (order_index == 0) ? enough_idle : 1;
if (find_next_cluster) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu,
&cpu_array[order_index][1], &has_misfit);
&cpu_array[order_index][1], &has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
}
@ -850,13 +887,13 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
if (order_index == 0) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
if (enough_idle) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
}
@ -866,52 +903,58 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
* cluster. In case no idle is found, pull it in.
*/
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
if (first_idle != -1)
if (first_idle != -1) {
walt_kick_cpu(first_idle);
else
} else if (walt_rotation_enabled) {
goto found_busy_cpu;
}
}
} else if (order_index == 2) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
/* help gold only if prime has had enough idle or gold has a misfit */
has_misfit = false;
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
/* help the farthest cluster indirectly if it needs help */
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1) {
first_idle =
find_first_idle_if_others_are_busy(&cpu_array[order_index][1]);
walt_kick_cpu(first_idle);
if (first_idle != -1) {
walt_kick_cpu(first_idle);
} else if (walt_rotation_enabled) {
goto found_busy_cpu;
}
}
} else {
busy_cpu =
walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit);
walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0],
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
if (enough_idle) {
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1)
goto found_busy_cpu;
}
has_misfit = false;
busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2],
&has_misfit);
&has_misfit, true);
if (busy_cpu != -1 && (enough_idle || has_misfit))
goto found_busy_cpu;
}
@ -1008,7 +1051,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu,
* remain same.
*/
cpumask_and(&src_mask, sched_group_span(group), env_cpus);
busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit);
busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit, false);
done:
if (busiest_cpu != -1)
*busiest = cpu_rq(busiest_cpu);

8
kernel/sched/walt/walt_rt.c
View File

@ -290,8 +290,8 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
lowest_mask, walt_rt_task_fits_capacity);
/* create a fastpath for finding a packing cpu */
packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task));
if (walt_choose_packing_cpu(packing_cpu, task)) {
packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task);
if (packing_cpu >= 0) {
*new_cpu = packing_cpu;
goto unlock;
}
@ -333,8 +333,8 @@ static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task,
return;
/* create a fastpath for finding a packing cpu */
packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task));
if (walt_choose_packing_cpu(packing_cpu, task)) {
packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task);
if (packing_cpu >= 0) {
*best_cpu = packing_cpu;
return;
}