github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-10-12 05:17:15 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug

Browse Source 
alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79f ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com
This commit is contained in:
Tim Chen 2026-09-21 17:37:22 -07:00 committed by Ingo Molnar
parent 93f51579e7
commit 0d6526f82c
1 changed files with 58 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
kernel/sched/fair.c
View File

@ -1546,6 +1546,28 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
(scale * per_cpu(sd_llc_size, cpu)));
}
/*
* A task counts in nr_pref_llc_running while it is queued on its preferred
* LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in
* the runnable domain so alb_break_llc() can compare it with h_nr_runnable.
*/
static bool task_pref_llc_runnable(struct task_struct *p)
{
return p->pref_llc_queued && !p->se.sched_delayed;
}
static void pref_llc_running_inc(struct rq *rq, struct task_struct *p)
{
if (task_pref_llc_runnable(p))
rq->nr_pref_llc_running++;
}
static void pref_llc_running_dec(struct rq *rq, struct task_struct *p)
{
if (task_pref_llc_runnable(p))
rq->nr_pref_llc_running--;
}
static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
{
int pref_llc, pref_llc_queued;
@ -1557,7 +1579,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
pref_llc_queued = (pref_llc == task_llc(p));
rq->nr_llc_running++;
rq->nr_pref_llc_running += pref_llc_queued;
/*
* Record whether p is enqueued on its preferred
@ -1575,6 +1596,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
*/
p->pref_llc_queued = pref_llc_queued;
/* Skipped while delayed; clear_delayed() adds it back on wake. */
pref_llc_running_inc(rq, p);
sd = rcu_dereference_all(rq->sd);
if (sd && (unsigned int)pref_llc < sd->llc_max)
sd->llc_counts[pref_llc]++;
@ -1591,7 +1615,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
rq->nr_llc_running--;
if (p->pref_llc_queued) {
rq->nr_pref_llc_running--;
/*
* Skipped if still delayed (set_delayed() already removed it);
* clearing pref_llc_queued below also stops clear_delayed()
* from re-adding it.
*/
pref_llc_running_dec(rq, p);
/*
* Update the status in case
* other logic might query
@ -1995,6 +2024,7 @@ void init_sched_mm(struct task_struct *p)
* polluting account_llc_enqueue().
*/
p->preferred_llc = -1;
p->pref_llc_queued = 0;
}
#else /* CONFIG_SCHED_CACHE */
@ -2016,6 +2046,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {}
static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {}
#endif /* CONFIG_SCHED_CACHE */
/*
@ -6390,15 +6424,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
static void set_delayed(struct sched_entity *se)
{
se->sched_delayed = 1;
/*
* Delayed se of cfs_rq have no tasks queued on them.
* Do not adjust h_nr_runnable since __dequeue_task()
* will account it for blocked tasks.
*
* This check can be removed because when flat pick
* patches get merged as only task can get delayed,
* same for clear_delayed().
*/
if (!entity_is_task(se))
if (!entity_is_task(se)) {
se->sched_delayed = 1;
return;
}
/*
* Drop a task leaving the runnable set.
* Needs to be called before sched_delayed is set.
* clear_delayed() mirrors this after clearing the flag.
*/
pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se));
se->sched_delayed = 1;
for_each_sched_entity(se) {
struct cfs_rq *cfs_rq = cfs_rq_of(se);
@ -6420,6 +6466,13 @@ static void clear_delayed(struct sched_entity *se)
if (!entity_is_task(se))
return;
/*
* Re-add on wake, after sched_delayed is cleared. On a final delayed
* dequeue account_llc_dequeue() already cleared pref_llc_queued, so
* this does nothing.
*/
pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se));
for_each_sched_entity(se) {
struct cfs_rq *cfs_rq = cfs_rq_of(se);