github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched_ext: Generalize the reject DSQ reenqueue path

Browse Source 
The reject DSQ is currently specific to sub-scheduler cap failures. Its
storage and initialization depend on CONFIG_EXT_SUB_SCHED, and the drain
path assumes every rejected task was rejected for SCX_TASK_REENQ_CAP.

Other transient placement failures need the same ability to park a task
on its source rq and return it to the owning BPF scheduler. Make the
reject DSQ unconditional and carry the reenqueue reason directly in
p->scx.flags from the rejection site.

A reenqueue reason remains valid while ops.enqueue() runs so that BPF
can inspect it. Clear it immediately after the callback returns and
before resolving a direct dispatch. Paths which bypass ops.enqueue()
clear the reason before the kernel-selected placement. A rejection from
the new placement can therefore install its reason into a clear field.
Route all paths through a common cleanup point to enforce this ordering.

This is a preparatory change to support proxy execution with sched_ext.

Signed-off-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Andrea Righi 2026-09-22 18:51:48 +02:00 committed by Tejun Heo
parent cea8b60e4a
commit 4eeabe0a3e
3 changed files with 53 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
kernel/sched/ext/ext.c
View File

@ -1672,12 +1672,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, struct task_struct *p,
u64 slice, u64 vtime, u64 enq_flags)
{
bool is_rq_owned = false;
bool is_rq_owned = dsq_is_rq_owned(dsq);
if (dsq->id == SCX_DSQ_LOCAL) {
if (dsq->id == SCX_DSQ_LOCAL)
dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
is_rq_owned = true;
}
WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
@ -2065,6 +2063,12 @@ bool scx_rq_online(struct rq *rq)
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
int sticky_cpu)
{
enum {
ENQ_ACTION_NONE,
ENQ_ACTION_DIRECT,
ENQ_ACTION_LOCAL_NOREFILL,
ENQ_ACTION_ENQUEUE,
} action = ENQ_ACTION_NONE;
struct scx_sched *sch = scx_task_sched(p);
struct task_struct **ddsp_taskp;
struct scx_dispatch_q *dsq;
@ -2098,7 +2102,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
__scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
"%s[%d] reenqueued %u times without running",
p->comm, p->pid, p->scx.reenq_cnt);
return;
goto out;
}
}
@ -2163,14 +2167,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
* dequeue may be waiting. The store_release matches their load_acquire.
*/
atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
return;
goto out;
direct:
direct_dispatch(sch, p, enq_flags);
return;
action = ENQ_ACTION_DIRECT;
goto out;
local_norefill:
scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
return;
action = ENQ_ACTION_LOCAL_NOREFILL;
goto out;
local:
dsq = &rq->scx.local_dsq;
goto enqueue;
@ -2182,9 +2186,27 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
goto enqueue;
enqueue:
refill_task_slice_dfl(sch, p);
clear_direct_dispatch(p);
scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
action = ENQ_ACTION_ENQUEUE;
out:
/* The reason is input to ops.enqueue(), not to the resulting placement. */
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
switch (action) {
case ENQ_ACTION_DIRECT:
direct_dispatch(sch, p, enq_flags);
break;
case ENQ_ACTION_LOCAL_NOREFILL:
scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
enq_flags);
break;
case ENQ_ACTION_ENQUEUE:
refill_task_slice_dfl(sch, p);
clear_direct_dispatch(p);
scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
break;
case ENQ_ACTION_NONE:
break;
}
}
static bool task_runnable(const struct task_struct *p)
@ -2410,6 +2432,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
sub_nr_running(rq, 1);
scx_dispatch_dequeue(rq, p);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
/* see scx_task_slice_ended() for the save/restore exception */
if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
@ -3140,6 +3163,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t
*/
ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
scx_dispatch_dequeue(rq, p);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
}
p->se.exec_start = rq_clock_task(rq);
@ -3314,7 +3338,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
if (p->scx.flags & SCX_TASK_IMMED) {
p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
} else {
u64 enq_flags = 0;
@ -4655,8 +4678,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
scx_reenq_wait_dispatching(p);
scx_dispatch_dequeue(rq, p);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= reason;
list_add_tail(&p->scx.dsq_list.node, &tasks);
@ -4667,7 +4689,6 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
nr_enqueued++;
}
@ -4780,14 +4801,11 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
dispatch_dequeue_locked(p, dsq);
raw_spin_unlock(&dsq->lock);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= reason;
scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
scx_rq_lock_drop(locked_rq);
raw_spin_rq_unlock(locked_rq);
@ -4849,14 +4867,14 @@ static void process_deferred_reenq_users(struct rq *rq)
}
}
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
* from p->scx.reenq_reason_*.
* Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF
* scheduler chooses placement again.
*
* A task can be re-rejected repeatedly. The reenqueue is bounded per task in
* scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT.
* Rejection can't happen for root.
* A task can be re-rejected repeatedly. Reenqueues are bounded per task by
* SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning
* scheduler. The private list below prevents a task from being revisited in
* the same round.
*/
static void scx_reenq_reject(struct rq *rq)
{
@ -4865,25 +4883,21 @@ static void scx_reenq_reject(struct rq *rq)
lockdep_assert_rq_held(rq);
if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
if (list_empty(&rq->scx.reject_dsq.list))
return;
/*
* Move to a private list so a task re-rejected by the
* Move tasks to a private list so a task re-rejected by
* scx_do_enqueue_task() below isn't revisited this round.
*/
list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
/* migration_pending tasks should have bypassed to local DSQ */
if (WARN_ON_ONCE(p->migration_pending))
continue;
WARN_ON_ONCE(p->migration_pending);
WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK));
scx_reenq_wait_dispatching(p);
scx_dispatch_dequeue(rq, p);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
p->scx.flags |= SCX_TASK_REENQ_CAP;
list_add_tail(&p->scx.dsq_list.node, &tasks);
}
@ -4891,13 +4905,8 @@ static void scx_reenq_reject(struct rq *rq)
list_del_init(&p->scx.dsq_list.node);
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
}
}
#else
static void scx_reenq_reject(struct rq *rq) {}
#endif
static void run_deferred(struct rq *rq)
{
@ -9015,8 +9024,8 @@ void __init init_sched_ext_class(void)
/* local_dsq's sch will be set during scx_root_enable() */
BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
#ifdef CONFIG_EXT_SUB_SCHED
BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
#ifdef CONFIG_EXT_SUB_SCHED
scx_rescue_init(rq);
#endif

2
kernel/sched/ext/sub.c
View File

@ -750,6 +750,8 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq
p->scx.reenq_reason_caps = missing;
p->scx.reenq_reason_cid = cid;
WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= SCX_TASK_REENQ_CAP;
return &rq->scx.reject_dsq;
}

2
kernel/sched/sched.h
View File

@ -810,8 +810,8 @@ struct scx_rq_rescue {
struct scx_rq {
struct scx_dispatch_q local_dsq;
struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */
struct scx_rq_rescue rescue;
#endif
struct list_head runnable_list; /* runnable tasks on this rq */