diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 0eab97e74b21..8a213e937820 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -93,6 +93,7 @@ struct walt_task_struct { * 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions * 0 = nothing, 1 = enqueued, 2 = dequeued */ + u32 flags; u64 mark_start; u64 window_start; u32 sum, demand; @@ -141,6 +142,16 @@ struct walt_task_struct { u8 hung_detect_status; }; +/* + * enumeration to set the flags variable + * each index below represents an offset into + * wts->flags + */ +enum walt_flags { + WALT_INIT, + MAX_WALT_FLAGS +}; + #define wts_to_ts(wts) ({ \ void *__mptr = (void *)(wts); \ ((struct task_struct *)(__mptr - \ diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 7de85593f288..f36038918132 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -296,8 +296,14 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; bool is_hiload; + bool is_ed_boost = wg_cpu->walt_load.ed_active; unsigned long pl = wg_cpu->walt_load.pl; + if (is_ed_boost) { + cpu_util = mult_frac(cpu_util, 100 + sysctl_ed_boost_pct, 100); + max_and_reason(util, cpu_util, wg_cpu, CPUFREQ_REASON_EARLY_DET); + } + if (is_rtg_boost) max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST); @@ -316,6 +322,9 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut pl = mult_frac(pl, TARGET_LOAD, 100); max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); } + + if (is_ed_boost) + wg_cpu->reasons |= CPUFREQ_REASON_EARLY_DET; } static inline unsigned long target_util(struct waltgov_policy *wg_policy, diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 310956da94e0..0e8439d33799 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -74,6 +74,7 @@ unsigned int sysctl_sched_asymcap_boost; unsigned int sysctl_sched_long_running_rt_task_ms; unsigned int sysctl_sched_idle_enough; unsigned int sysctl_sched_cluster_util_thres_pct; +unsigned int sysctl_ed_boost_pct; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -922,6 +923,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &two_thousand, }, + { + .procname = "sched_ed_boost", + .data = &sysctl_ed_boost_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred, + }, { } }; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 18dd18f34135..21f86b54a5df 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -354,79 +354,6 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, static void rollover_cpu_window(struct rq *rq, bool full_window); static void rollover_top_tasks(struct rq *rq, bool full_window); -/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. - * @start_cpu: The cpu from the cluster to choose from - * - * If the cluster has a 32bit capable cpu return it regardless - * of whether it is halted or not. - * - * If the cluster does not have a 32 bit capable cpu, find the - * first unhalted, active cpu in this cluster. - */ -int walt_find_cluster_packing_cpu(int start_cpu) -{ - struct rq *rq = cpu_rq(start_cpu); - struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; - struct walt_sched_cluster *cluster = wrq->cluster; - cpumask_t unhalted_cpus; - cpumask_t cluster_32bit_cpus; - - /* find all 32 bit capable cpus in this cluster */ - cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); - - /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ - if (cpumask_weight(&cluster_32bit_cpus) > 0) - return cpumask_first(&cluster_32bit_cpus); - - /* find all unhalted active cpus */ - cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); - - /* find all unhalted active cpus in this cluster */ - cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); - - /* return the first found unhalted, active cpu, in this cluster */ - return cpumask_first(&unhalted_cpus); -} - -/* for cfs and rt, determine if packing_cpu should be used */ -bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) -{ - struct rq *rq; - struct walt_rq *wrq; - struct walt_sched_cluster *cluster; - - /* packing cpu must be a valid cpu for runqueue lookup */ - if (packing_cpu >= nr_cpu_ids) - return false; - - rq = cpu_rq(packing_cpu); - wrq = (struct walt_rq *)rq->android_vendor_data1; - cluster = wrq->cluster; - - /* if idle_enough feature is not enabled */ - if (!sysctl_sched_idle_enough) - return false; - - /* if cpu is not allowed for this task */ - if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) - return false; - - /* if cluster util is high */ - if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) - return false; - - /* if cpu utilization is high */ - if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) - return false; - - /* don't pack big tasks */ - if (task_util(p) >= sysctl_sched_idle_enough) - return false; - - /* the packing cpu can be used, so pack! */ - return true; -} - /* * Demand aggregation for frequency purpose: * @@ -657,12 +584,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) u64 load, tt_load = 0, kload = 0; struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); - if (wrq->ed_task != NULL) { - load = sched_ravg_window; - *reason = CPUFREQ_REASON_EARLY_DET; - goto done; - } - if (sched_freq_aggr_en) { load = wrq->prev_runnable_sum + aggr_grp_load; *reason = CPUFREQ_REASON_FREQ_AGR; @@ -694,7 +615,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) *reason = CPUFREQ_REASON_SUH; } -done: trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, load, 0, walt_rotation_enabled, sysctl_sched_user_hint, wrq, *reason); @@ -732,6 +652,10 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea walt_load->pl = pl; walt_load->ws = walt_load_reported_window; walt_load->rtgb_active = rtgb_active; + if (wrq->ed_task) + walt_load->ed_active = true; + else + walt_load->ed_active = false; } return (util >= capacity) ? capacity : util; @@ -2457,6 +2381,7 @@ static void init_new_task_load(struct task_struct *p) wts->mvp_prio = WALT_NOT_MVP; wts->cidx = 0; __sched_fork_init(p); + walt_flag_set(p, WALT_INIT, 1); } static void init_existing_task_load(struct task_struct *p) @@ -3798,6 +3723,50 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) } } +static void update_cpu_capacity_helper(int cpu) +{ + unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); + unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); + unsigned long thermal_cap, old; + struct walt_sched_cluster *cluster; + struct rq *rq = cpu_rq(cpu); + + if (unlikely(walt_disabled)) + return; + + /* + * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. + * so, + * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. + */ + + thermal_cap = fmax_capacity - thermal_pressure; + + cluster = cpu_cluster(cpu); + /* reduce the fmax_capacity under cpufreq constraints */ + if (cluster->max_freq != cluster->max_possible_freq) + fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, + cluster->max_possible_freq); + + old = rq->cpu_capacity_orig; + rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); + + if (old != rq->cpu_capacity_orig) + trace_update_cpu_capacity(cpu, 0, 0); +} + +/* + * The intention of this hook is to update cpu_capacity_orig as well as + * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). + */ +static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) +{ + unsigned long rt_pressure = arch_scale_cpu_capacity(cpu) - *capacity; + + update_cpu_capacity_helper(cpu); + *capacity = max((int)(cpu_rq(cpu)->cpu_capacity_orig - rt_pressure), 0); +} + /** * walt_irq_work() - perform walt irq work for rollover and migration * @@ -3839,6 +3808,11 @@ static void walt_irq_work(struct irq_work *irq_work) level++; } + if (!is_migration) { + for_each_cpu(cpu, &lock_cpus) + update_cpu_capacity_helper(cpu); + } + __walt_irq_work_locked(is_migration, &lock_cpus); for_each_cpu(cpu, &lock_cpus) @@ -4074,45 +4048,6 @@ static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *polic cpu_cluster(policy->cpu)->max_freq = policy->max; } -/* - * The intention of this hook is to update cpu_capacity_orig as well as - * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). - */ -static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) -{ - unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); - unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); - unsigned long thermal_cap, old; - unsigned long rt_pressure = fmax_capacity - *capacity; - struct walt_sched_cluster *cluster; - struct rq *rq = cpu_rq(cpu); - - if (unlikely(walt_disabled)) - return; - - /* - * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. - * so, - * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. - */ - - thermal_cap = fmax_capacity - thermal_pressure; - - cluster = cpu_cluster(cpu); - /* reduce the fmax_capacity under cpufreq constraints */ - if (cluster->max_freq != cluster->max_possible_freq) - fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, - cluster->max_possible_freq); - - old = rq->cpu_capacity_orig; - rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); - - if (old != rq->cpu_capacity_orig) - trace_update_cpu_capacity(cpu, rt_pressure, *capacity); - - *capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL); -} - static void android_rvh_sched_cpu_starting(void *unused, int cpu) { if (unlikely(walt_disabled)) @@ -4255,6 +4190,10 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (!double_enqueue) walt_inc_cumulative_runnable_avg(rq, p); + + if ((flags & ENQUEUE_WAKEUP) && do_pl_notif(rq)) + waltgov_run_callback(rq, WALT_CPUFREQ_PL); + trace_sched_enq_deq_task(p, 1, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); } @@ -4274,7 +4213,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st * therefore the check to ensure that prev_on_rq_cpu is needed to prevent * an invalid failure. */ - if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) + if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq) && + walt_flag_test(p, WALT_INIT)) WALT_BUG(WALT_BUG_UPSTREAM, p, "dequeue cpu %d not same as enqueue %d\n", cpu_of(rq), wts->prev_on_rq_cpu); @@ -4371,20 +4311,6 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) rcu_read_unlock(); } -static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct *p) -{ - unsigned long flags; - int cpu = p->cpu; - - if (unlikely(walt_disabled)) - return; - - raw_spin_lock_irqsave(&cpu_rq(cpu)->__lock, flags); - if (do_pl_notif(cpu_rq(cpu))) - waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL); - raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); -} - static u64 tick_sched_clock; static DECLARE_COMPLETION(tick_sched_clock_completion); @@ -4556,7 +4482,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL); register_trace_android_rvh_after_dequeue_task(android_rvh_dequeue_task, NULL); register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); - register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL); register_trace_android_rvh_schedule(android_rvh_schedule, NULL); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index b8a0e5bf8096..0e547b69a0c5 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -59,6 +59,7 @@ struct walt_cpu_load { unsigned long pl; bool rtgb_active; u64 ws; + bool ed_active; }; #define DECLARE_BITMAP_ARRAY(name, nr, bits) \ @@ -216,6 +217,7 @@ extern unsigned int sysctl_sched_user_hint; extern unsigned int sysctl_sched_conservative_pl; extern unsigned int sysctl_sched_hyst_min_coloc_ns; extern unsigned int sysctl_sched_long_running_rt_task_ms; +extern unsigned int sysctl_ed_boost_pct; #define WALT_MANY_WAKEUP_DEFAULT 1000 extern unsigned int sysctl_sched_many_wakeup_threshold; @@ -903,12 +905,31 @@ extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int wr void __user *buffer, size_t *lenp, loff_t *ppos); +static inline void walt_flag_set(struct task_struct *p, enum walt_flags feature, bool set) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (set) + wts->flags |= 1 << feature; + else + wts->flags &= ~(1 << feature); +} + +static inline bool walt_flag_test(struct task_struct *p, enum walt_flags feature) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return !!(wts->flags & (1 << feature)); +} + #define WALT_MVP_SLICE 3000000U #define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) +/* higher number, better priority */ #define WALT_RTG_MVP 0 #define WALT_BINDER_MVP 1 #define WALT_TASK_BOOST_MVP 2 +#define WALT_LL_PIPE_MVP 3 #define WALT_NOT_MVP -1 @@ -945,6 +966,74 @@ extern struct cpumask __cpu_halt_mask; #define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) #define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) +/* walt_find_and_choose_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. + * @start_cpu: The cpu from the cluster to choose from + * + * If the cluster has a 32bit capable cpu return it regardless + * of whether it is halted or not. + * + * If the cluster does not have a 32 bit capable cpu, find the + * first unhalted, active cpu in this cluster. + * + * Returns -1 if packing_cpu if not found or is unsuitable to be packed on to + * Returns a valid cpu number if packing_cpu is found and is usable + */ +static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p) +{ + struct rq *rq = cpu_rq(start_cpu); + struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + cpumask_t unhalted_cpus; + cpumask_t cluster_32bit_cpus; + int packing_cpu; + + /* if idle_enough feature is not enabled */ + if (!sysctl_sched_idle_enough) + return -1; + if (!sysctl_sched_cluster_util_thres_pct) + return -1; + + /* find all 32 bit capable cpus in this cluster */ + cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); + + /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ + if (cpumask_weight(&cluster_32bit_cpus) > 0) { + packing_cpu = cpumask_first(&cluster_32bit_cpus); + } else { + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + + /* return the first found unhalted, active cpu, in this cluster */ + packing_cpu = cpumask_first(&unhalted_cpus); + } + + /* packing cpu must be a valid cpu for runqueue lookup */ + if (packing_cpu >= nr_cpu_ids) + return -1; + + /* if cpu is not allowed for this task */ + if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) + return -1; + + /* if cluster util is high */ + if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) + return -1; + + /* if cpu utilization is high */ + if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) + return -1; + + /* don't pack big tasks */ + if (task_util(p) >= sysctl_sched_idle_enough) + return -1; + + /* the packing cpu can be used, so pack! */ + return packing_cpu; +} + extern void walt_task_dump(struct task_struct *p); extern void walt_rq_dump(int cpu); extern void walt_dump(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index da6489938e69..6a44c6c9400c 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -4,6 +4,7 @@ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include #include @@ -280,8 +281,8 @@ static void walt_find_best_target(struct sched_domain *sd, } /* fast path for packing_cpu */ - packing_cpu = walt_find_cluster_packing_cpu(start_cpu); - if (walt_choose_packing_cpu(packing_cpu, p)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(start_cpu, p); + if (packing_cpu >= 0) { fbt_env->fastpath = CLUSTER_PACKING_FASTPATH; cpumask_set_cpu(packing_cpu, candidates); goto out; @@ -1041,14 +1042,17 @@ static void binder_restore_priority_hook(void *data, */ int walt_get_mvp_task_prio(struct task_struct *p) { + if (walt_procfs_low_latency_task(p) || + walt_pipeline_low_latency_task(p)) + return WALT_LL_PIPE_MVP; + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) return WALT_TASK_BOOST_MVP; if (walt_binder_low_latency_task(p)) return WALT_BINDER_MVP; - if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || - walt_pipeline_low_latency_task(p)) + if (task_rtg_high_prio(p)) return WALT_RTG_MVP; return WALT_NOT_MVP; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index a3020210bbc3..95dd2e1b645e 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -296,17 +296,19 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; struct walt_task_struct *wts; + struct task_struct *pull_me; + int task_visited; BUG_ON(src_cpu == dst_cpu); to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; - to_higher = dst_wrq->cluster->id > src_wrq->cluster->id; raw_spin_lock_irqsave(&src_rq->__lock, flags); + pull_me = NULL; + task_visited = 0; list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { - if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; @@ -317,13 +319,30 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) false)) continue; - walt_detach_task(p, src_rq, dst_rq); - pulled_task = p; + if (pull_me == NULL) { + pull_me = p; + } else { + if (to_lower) { + if (task_util(p) < task_util(pull_me)) + pull_me = p; + } else if (task_util(p) > task_util(pull_me)) { + pull_me = p; + } + } + + task_visited++; + if (task_visited > 5) + break; + } + if (pull_me) { + walt_detach_task(pull_me, src_rq, dst_rq); + pulled_task = pull_me; goto unlock; } + pull_me = NULL; + task_visited = 0; list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { - if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; @@ -334,8 +353,24 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) true)) continue; - walt_detach_task(p, src_rq, dst_rq); - pulled_task = p; + if (pull_me == NULL) { + pull_me = p; + } else { + if (to_lower) { + if (task_util(p) < task_util(pull_me)) + pull_me = p; + } else if (task_util(p) > task_util(pull_me)) { + pull_me = p; + } + } + + task_visited++; + if (task_visited > 5) + break; + } + if (pull_me) { + walt_detach_task(pull_me, src_rq, dst_rq); + pulled_task = pull_me; goto unlock; } @@ -380,7 +415,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) return 0; raw_spin_lock_irqsave(&dst_rq->__lock, flags); - walt_attach_task(p, dst_rq); + walt_attach_task(pulled_task, dst_rq); raw_spin_unlock_irqrestore(&dst_rq->__lock, flags); return 1; /* we pulled 1 task */ @@ -426,7 +461,7 @@ static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) } static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -452,7 +487,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr } static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -512,7 +547,7 @@ static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t } static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -521,6 +556,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t int total_cpus = 0; int busy_nr_big_tasks = 0; struct walt_rq *wrq; + bool treat_dst_idle = is_newidle || available_idle_cpu(dst_cpu); /* * A higher capacity CPU is looking at a lower capacity @@ -552,7 +588,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t /* active migration is allowed only to idle cpu */ if (cpu_rq(i)->cfs.h_nr_running < 2 && - (!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu))) + (!wrq->walt_stats.nr_big_tasks || !treat_dst_idle)) continue; if (!walt_rotation_enabled && !cpu_overutilized(i) && @@ -579,7 +615,8 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t return busiest_cpu; } -static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit) +static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit, + bool is_newidle) { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; @@ -588,13 +625,13 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); else busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); return busiest_cpu; } @@ -830,7 +867,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, */ if (num_sched_clusters <= 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; @@ -839,7 +876,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, find_next_cluster = (order_index == 0) ? enough_idle : 1; if (find_next_cluster) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, - &cpu_array[order_index][1], &has_misfit); + &cpu_array[order_index][1], &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; } @@ -850,13 +887,13 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (order_index == 0) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; if (enough_idle) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; } @@ -866,52 +903,58 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * cluster. In case no idle is found, pull it in. */ busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - if (first_idle != -1) + if (first_idle != -1) { walt_kick_cpu(first_idle); - else + } else if (walt_rotation_enabled) { goto found_busy_cpu; + } } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; /* help gold only if prime has had enough idle or gold has a misfit */ has_misfit = false; busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; /* help the farthest cluster indirectly if it needs help */ busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - walt_kick_cpu(first_idle); + if (first_idle != -1) { + walt_kick_cpu(first_idle); + } else if (walt_rotation_enabled) { + goto found_busy_cpu; + } } } else { busy_cpu = - walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit); + walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; if (enough_idle) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; } has_misfit = false; busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; } @@ -1008,7 +1051,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, * remain same. */ cpumask_and(&src_mask, sched_group_span(group), env_cpus); - busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit); + busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit, false); done: if (busiest_cpu != -1) *busiest = cpu_rq(busiest_cpu); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index e69fe2a9e9de..3265be5ba664 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -290,8 +290,8 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c lowest_mask, walt_rt_task_fits_capacity); /* create a fastpath for finding a packing cpu */ - packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); - if (walt_choose_packing_cpu(packing_cpu, task)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task); + if (packing_cpu >= 0) { *new_cpu = packing_cpu; goto unlock; } @@ -333,8 +333,8 @@ static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, return; /* create a fastpath for finding a packing cpu */ - packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); - if (walt_choose_packing_cpu(packing_cpu, task)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task); + if (packing_cpu >= 0) { *best_cpu = packing_cpu; return; }