diff --git a/kernel/sched/core.c b/kernel/sched/core.c index acea9428769f..002e846e7db2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1198,6 +1198,40 @@ static void uclamp_fork(struct task_struct *p) } } +#ifdef CONFIG_SMP +unsigned int uclamp_task(struct task_struct *p) +{ + unsigned long util; + + util = task_util_est(p); + util = max(util, uclamp_eff_value(p, UCLAMP_MIN)); + util = min(util, uclamp_eff_value(p, UCLAMP_MAX)); + + return util; +} + +bool uclamp_boosted(struct task_struct *p) +{ + return uclamp_eff_value(p, UCLAMP_MIN) > 0; +} + +bool uclamp_latency_sensitive(struct task_struct *p) +{ +#ifdef CONFIG_UCLAMP_TASK_GROUP + struct cgroup_subsys_state *css = task_css(p, cpu_cgrp_id); + struct task_group *tg; + + if (!css) + return false; + tg = container_of(css, struct task_group, css); + + return tg->latency_sensitive; +#else + return false; +#endif +} +#endif /* CONFIG_SMP */ + static void __init init_uclamp(void) { struct uclamp_se uc_max = {}; @@ -1239,6 +1273,41 @@ static inline int uclamp_validate(struct task_struct *p, static void __setscheduler_uclamp(struct task_struct *p, const struct sched_attr *attr) { } static inline void uclamp_fork(struct task_struct *p) { } + +long schedtune_task_margin(struct task_struct *task); + +#ifdef CONFIG_SMP +unsigned int uclamp_task(struct task_struct *p) +{ + unsigned long util = task_util_est(p); +#ifdef CONFIG_SCHED_TUNE + long margin = schedtune_task_margin(p); + + trace_sched_boost_task(p, util, margin); + + util += margin; +#endif + + return util; +} + +bool uclamp_boosted(struct task_struct *p) +{ +#ifdef CONFIG_SCHED_TUNE + return schedtune_task_boost(p) > 0; +#endif + return false; +} + +bool uclamp_latency_sensitive(struct task_struct *p) +{ +#ifdef CONFIG_SCHED_TUNE + return schedtune_prefer_idle(p) != 0; +#endif + return false; +} +#endif /* CONFIG_SMP */ + static inline void init_uclamp(void) { } #endif /* CONFIG_UCLAMP_TASK */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5e445aeeb74b..abd572ea22d3 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3690,7 +3690,7 @@ static inline unsigned long _task_util_est(struct task_struct *p) return max(ue.ewma, ue.enqueued); } -static inline unsigned long task_util_est(struct task_struct *p) +unsigned long task_util_est(struct task_struct *p) { return max(task_util(p), _task_util_est(p)); } @@ -5851,8 +5851,7 @@ schedtune_cpu_margin(unsigned long util, int cpu) return schedtune_margin(util, boost); } -static inline long -schedtune_task_margin(struct task_struct *task) +long schedtune_task_margin(struct task_struct *task) { int boost = schedtune_task_boost(task); unsigned long util; @@ -5887,25 +5886,8 @@ schedtune_cpu_margin(unsigned long util, int cpu) return 0; } -static inline int -schedtune_task_margin(struct task_struct *task) -{ - return 0; -} - #endif /* CONFIG_SCHED_TUNE */ -static inline unsigned long -boosted_task_util(struct task_struct *task) -{ - unsigned long util = task_util_est(task); - long margin = schedtune_task_margin(task); - - trace_sched_boost_task(task, util, margin); - - return util + margin; -} - static unsigned long cpu_util_without(int cpu, struct task_struct *p); static unsigned long capacity_spare_without(int cpu, struct task_struct *p) @@ -6553,13 +6535,11 @@ unsigned long capacity_curr_of(int cpu) static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, struct task_struct *p) { - unsigned long min_util = boosted_task_util(p); + unsigned long min_util = uclamp_task(p); unsigned long target_capacity = ULONG_MAX; unsigned long min_wake_util = ULONG_MAX; unsigned long target_max_spare_cap = 0; unsigned long target_util = ULONG_MAX; - bool prefer_idle = schedtune_prefer_idle(p); - bool boosted = schedtune_task_boost(p) > 0; /* Initialise with deepest possible cstate (INT_MAX) */ int shallowest_idle_cstate = INT_MAX; struct sched_group *sg; @@ -6567,6 +6547,8 @@ static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, int best_idle_cpu = -1; int target_cpu = -1; int backup_cpu = -1; + bool prefer_idle; + bool boosted; int i; /* @@ -6578,6 +6560,8 @@ static void find_best_target(struct sched_domain *sd, cpumask_t *cpus, * performance CPU, thus requiring to maximise target_capacity. In this * case we initialise target_capacity to 0. */ + prefer_idle = uclamp_latency_sensitive(p); + boosted = uclamp_boosted(p); if (prefer_idle && boosted) target_capacity = 0; @@ -6989,8 +6973,8 @@ static void select_cpu_candidates(struct sched_domain *sd, cpumask_t *cpus, { int highest_spare_cap_cpu = prev_cpu, best_idle_cpu = -1; unsigned long spare_cap, max_spare_cap, util, cpu_cap; - bool prefer_idle = schedtune_prefer_idle(p); - bool boosted = schedtune_task_boost(p) > 0; + bool prefer_idle = uclamp_latency_sensitive(p); + bool boosted = uclamp_boosted(p); unsigned long target_cap = boosted ? 0 : ULONG_MAX; unsigned long highest_spare_cap = 0; unsigned int min_exit_lat = UINT_MAX; @@ -7151,7 +7135,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy /* If there is only one sensible candidate, select it now. */ cpu = cpumask_first(candidates); - if (weight == 1 && ((schedtune_prefer_idle(p) && idle_cpu(cpu)) || + if (weight == 1 && ((uclamp_latency_sensitive(p) && idle_cpu(cpu)) || (cpu == prev_cpu))) { best_energy_cpu = cpu; goto unlock; @@ -7219,7 +7203,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f record_wakee(p); if (static_branch_unlikely(&sched_energy_present)) { - if (schedtune_prefer_idle(p) && !sched_feat(EAS_PREFER_IDLE) && !sync) + if (uclamp_latency_sensitive(p) && !sched_feat(EAS_PREFER_IDLE) && !sync) goto sd_loop; new_cpu = find_energy_efficient_cpu(p, prev_cpu, sync); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4b704e11bed4..09d220d6be05 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2354,6 +2354,11 @@ unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util, } #endif /* CONFIG_UCLAMP_TASK */ +unsigned long task_util_est(struct task_struct *p); +unsigned int uclamp_task(struct task_struct *p); +bool uclamp_latency_sensitive(struct task_struct *p); +bool uclamp_boosted(struct task_struct *p); + #ifdef arch_scale_freq_capacity # ifndef arch_scale_freq_invariant # define arch_scale_freq_invariant() true