diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a4b685d554c2..9a4a5afee211 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1672,12 +1672,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - bool is_rq_owned = false; + bool is_rq_owned = dsq_is_rq_owned(dsq); - if (dsq->id == SCX_DSQ_LOCAL) { + if (dsq->id == SCX_DSQ_LOCAL) dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); - is_rq_owned = true; - } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -2065,6 +2063,12 @@ bool scx_rq_online(struct rq *rq) void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { + enum { + ENQ_ACTION_NONE, + ENQ_ACTION_DIRECT, + ENQ_ACTION_LOCAL_NOREFILL, + ENQ_ACTION_ENQUEUE, + } action = ENQ_ACTION_NONE; struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; @@ -2098,7 +2102,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), "%s[%d] reenqueued %u times without running", p->comm, p->pid, p->scx.reenq_cnt); - return; + goto out; } } @@ -2163,14 +2167,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; + goto out; direct: - direct_dispatch(sch, p, enq_flags); - return; + action = ENQ_ACTION_DIRECT; + goto out; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); - return; + action = ENQ_ACTION_LOCAL_NOREFILL; + goto out; local: dsq = &rq->scx.local_dsq; goto enqueue; @@ -2182,9 +2186,27 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, goto enqueue; enqueue: - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + action = ENQ_ACTION_ENQUEUE; +out: + /* The reason is input to ops.enqueue(), not to the resulting placement. */ + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + switch (action) { + case ENQ_ACTION_DIRECT: + direct_dispatch(sch, p, enq_flags); + break; + case ENQ_ACTION_LOCAL_NOREFILL: + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); + break; + case ENQ_ACTION_ENQUEUE: + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + break; + case ENQ_ACTION_NONE: + break; + } } static bool task_runnable(const struct task_struct *p) @@ -2410,6 +2432,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) @@ -3140,6 +3163,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } p->se.exec_start = rq_clock_task(rq); @@ -3314,7 +3338,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { u64 enq_flags = 0; @@ -4655,8 +4678,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; list_add_tail(&p->scx.dsq_list.node, &tasks); @@ -4667,7 +4689,6 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } @@ -4780,14 +4801,11 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); @@ -4849,14 +4867,14 @@ static void process_deferred_reenq_users(struct rq *rq) } } -#ifdef CONFIG_EXT_SUB_SCHED /* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. + * Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF + * scheduler chooses placement again. * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. + * A task can be re-rejected repeatedly. Reenqueues are bounded per task by + * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning + * scheduler. The private list below prevents a task from being revisited in + * the same round. */ static void scx_reenq_reject(struct rq *rq) { @@ -4865,25 +4883,21 @@ static void scx_reenq_reject(struct rq *rq) lockdep_assert_rq_held(rq); - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* - * Move to a private list so a task re-rejected by the + * Move tasks to a private list so a task re-rejected by * scx_do_enqueue_task() below isn't revisited this round. */ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; + WARN_ON_ONCE(p->migration_pending); + WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)); scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - list_add_tail(&p->scx.dsq_list.node, &tasks); } @@ -4891,13 +4905,8 @@ static void scx_reenq_reject(struct rq *rq) list_del_init(&p->scx.dsq_list.node); scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } } -#else -static void scx_reenq_reject(struct rq *rq) {} -#endif static void run_deferred(struct rq *rq) { @@ -9015,8 +9024,8 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); -#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED scx_rescue_init(rq); #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index abe10a3529b9..b5989aaffcc6 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -750,6 +750,8 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq p->scx.reenq_reason_caps = missing; p->scx.reenq_reason_cid = cid; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; return &rq->scx.reject_dsq; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d41d23a5632b..1df76a3f38b9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -810,8 +810,8 @@ struct scx_rq_rescue { struct scx_rq { struct scx_dispatch_q local_dsq; + struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */ #ifdef CONFIG_EXT_SUB_SCHED - struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */