diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index df8c9c2c7918..a24dd2089ae9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -13962,29 +13962,62 @@ static inline int on_null_domain(struct rq *rq) */ static inline int find_new_ilb(void) { - int this_cpu = smp_processor_id(); - const struct cpumask *hk_mask; - int ilb_cpu; + struct cpumask *ilb_cpus; + int ilb_cpu, fallback = -1; - hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); + lockdep_assert_irqs_disabled(); - for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { - if (ilb_cpu == this_cpu) + /* + * Reuse the per-CPU select_rq_mask, which is protected from concurrent + * use on this CPU by having interrupts disabled. + */ + ilb_cpus = this_cpu_cpumask_var_ptr(select_rq_mask); + cpumask_and(ilb_cpus, nohz.idle_cpus_mask, + housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)); + + for_each_cpu(ilb_cpu, ilb_cpus) { + if (!idle_cpu(ilb_cpu)) { + /* + * Once an idle fallback exists, a busy CPU proves that + * this core cannot be fully idle. Skip its siblings. + */ + if (sched_smt_active() && fallback >= 0) + cpumask_andnot(ilb_cpus, ilb_cpus, cpu_smt_mask(ilb_cpu)); continue; + } - if (idle_cpu(ilb_cpu)) - return ilb_cpu; + /* + * Running the idle load balancer on an idle sibling of a busy + * SMT core can reduce the capacity available to its sibling. Prefer + * a CPU whose entire core is idle, but retain the first idle CPU as + * a fallback so idle balancing can still make progress when no fully + * idle core exists. + */ + if (sched_smt_active() && !is_core_idle(ilb_cpu)) { + if (fallback < 0) + fallback = ilb_cpu; + + /* + * The core is not idle, so there is no need to check + * any of its other SMT siblings. + */ + cpumask_andnot(ilb_cpus, ilb_cpus, + cpu_smt_mask(ilb_cpu)); + continue; + } + + return ilb_cpu; } - return -1; + return fallback; } /* * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU * SMP function call (IPI). * - * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set - * (if there is one). + * Prefer a CPU on a fully idle core in the HK_TYPE_KERNEL_NOISE housekeeping + * set. Fall back to the first idle CPU when no fully idle core exists. */ static void kick_ilb(unsigned int flags) {