diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1c687c3c70f1..b42a0ca2a69e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8575,6 +8575,35 @@ static inline bool test_idle_cores(int cpu) return false; } +/* + * Redirect a CPU to a higher-priority available sibling in its SMT domain, + * subject to task affinity. + */ +static inline int select_idle_smt_cpu(struct task_struct *p, int cpu) +{ + struct sched_domain *sd; + int best = cpu; + int sibling; + + if (!sched_smt_active()) + return cpu; + + sd = rcu_dereference_all(cpu_rq(cpu)->sd); + if (!sd || !(sd->flags & SD_SHARE_CPUCAPACITY) || + !(sd->flags & SD_ASYM_PACKING)) + return cpu; + + for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) { + if (sibling == best || !choose_idle_cpu(sibling, p)) + continue; + + if (sched_asym_prefer(sibling, best)) + best = sibling; + } + + return best; +} + /* * Scans the local SMT mask to see if the entire core is idle, and records this * information in sd_balance_shared->has_idle_cores. @@ -8959,7 +8988,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (choose_idle_cpu(target, p) && asym_fits_cpu(task_util, util_min, util_max, target)) - return target; + goto select_smt_priority; /* * If the previous CPU is cache affine and idle, don't be stupid: @@ -8969,8 +8998,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, prev)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(prev, target)) - return prev; + cpus_share_resources(prev, target)) { + target = prev; + goto select_smt_priority; + } prev_aff = prev; } @@ -8988,7 +9019,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) prev == smp_processor_id() && this_rq()->nr_running <= 1 && asym_fits_cpu(task_util, util_min, util_max, prev)) { - return prev; + target = prev; + goto select_smt_priority; } /* Check a recently used CPU as a potential idle candidate: */ @@ -9002,8 +9034,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(recent_used_cpu, target)) - return recent_used_cpu; + cpus_share_resources(recent_used_cpu, target)) { + target = recent_used_cpu; + goto select_smt_priority; + } } else { recent_used_cpu = -1; @@ -9025,7 +9059,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (sd) { i = select_idle_capacity(p, sd, target); - return ((unsigned)i < nr_cpumask_bits) ? i : target; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } + return target; } } @@ -9038,14 +9076,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (!has_idle_core && cpus_share_cache(prev, target)) { i = select_idle_smt(p, sd, prev); - if ((unsigned int)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } } } i = select_idle_cpu(p, sd, has_idle_core, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } /* * For cluster machines which have lower sharing cache like L2 or @@ -9053,12 +9095,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * first. But prev_cpu or recent_used_cpu may also be a good candidate, * use them if possible when no idle CPU found in select_idle_cpu(). */ - if ((unsigned int)prev_aff < nr_cpumask_bits) - return prev_aff; - if ((unsigned int)recent_used_cpu < nr_cpumask_bits) - return recent_used_cpu; + if ((unsigned int)prev_aff < nr_cpumask_bits) { + target = prev_aff; + goto select_smt_priority; + } + if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { + target = recent_used_cpu; + goto select_smt_priority; + } return target; + +select_smt_priority: + return select_idle_smt_cpu(p, target); } /** @@ -9735,8 +9784,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) } /* Slow path */ - if (unlikely(sd)) - return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + if (unlikely(sd)) { + new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + return select_idle_smt_cpu(p, new_cpu); + } /* Fast path */ if (wake_flags & WF_TTWU)