From 7de9a6fb44eae4f05e68c805d58b9c618815adfa Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 15 Sep 2026 12:34:44 -1000 Subject: [PATCH 1/7] sched_ext: Wait for SCX_OPSS_DISPATCHING before reenqueueing a task ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") moved the final ops_state store in scx_dispatch_enqueue() after the DSQ unlock so that the custody update and ops.dequeue() precede it. A task can thus be found on a DSQ while still SCX_OPSS_DISPATCHING. The dequeue and core-sched pick paths wait for the state to clear in ops_dequeue() but the reenqueue paths don't. A reenqueue in that window runs ops.enqueue() and sets SCX_OPSS_QUEUED before the dispatch has completed. The dispatcher's final store then overwrites it with SCX_OPSS_NONE and finish_dispatch() drops every later dispatch of the task. Wait for SCX_OPSS_DISPATCHING to clear before dequeueing a task for reenqueue, the same way ops_dequeue() does. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.c | 1 + 3 files changed, 15 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 70b711c4de6e..725b890fcab9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4404,6 +4404,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p + * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p + * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + */ +void scx_reenq_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,6 +4458,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4570,6 +4582,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_reenq_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f113c1..115c96fbf932 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2078,6 +2078,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_reenq_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index f7aeb1488566..a30c965b175d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -801,6 +801,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) From df5cdc2c832ca4e8a6d774596b9005558761a403 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 16 Sep 2026 12:00:24 -1000 Subject: [PATCH 2/7] sched_ext: Derive SCX_RQ_IN_WAKEUP from the core enqueue flags schedule_deferred_locked() skips scheduling a deferred action while SCX_RQ_IN_WAKEUP is set and relies on the task_woken_scx() call that follows a wakeup enqueue to run it. enqueue_task_scx() sets the flag from the merged enqueue flags, which include the flags stashed for a remote activation. move_remote_task_to_local_dsq() thus sets SCX_RQ_IN_WAKEUP on the destination rq when the moved task was woken up, although no task_woken_scx() follows that activation. An IMMED insert into a busy destination requests a local reenqueue during that enqueue. The request gets linked but not scheduled and stays pending until an unrelated wakeup or preemption on that CPU runs the deferred actions. The IMMED task sits behind the running task in the meantime. If nothing runs them before the scheduler is disabled, the request outlives the scheduler and points into its freed per-cpu area, which the next scheduler dereferences from run_deferred(). Test the core enqueue flags for the wakeup bit. Only the core's wakeup path is followed by task_woken_scx(). Fixes: 57ccf5ccdc56 ("sched_ext: Fix enqueue_task_scx() truncation of upper enqueue flags") Cc: stable@vger.kernel.org # v7.1+ Reported-by: Andrea Righi Link: https://lore.kernel.org/all/20260916145807.3250167-1-arighi@nvidia.com/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 725b890fcab9..e207ccd17164 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2141,7 +2141,12 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* From cb86607ada73185f321baa7f9d94a08a3a40bbf5 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Thu, 17 Sep 2026 15:52:41 +0800 Subject: [PATCH 3/7] sched_ext: Don't run ops.dequeue() with a DSQ lock held ops.dequeue() is invoked with the source user DSQ's lock still held on the consume and move paths (scx_consume_dispatch_q(), move_task_between_dsqs()). A BPF scheduler which locks the source user DSQ from ops.dequeue() - e.g. by iterating it with bpf_iter_scx_dsq - self-deadlocks. ops.dequeue() can only call the "any" kfuncs and none of them can lock a builtin DSQ, so the global and bypass paths can't deadlock; however, all DSQ locks share one lockdep class, so iterating any user DSQ from ops.dequeue() on those paths trips the recursion check. Move the invocation after the DSQ unlock on all three paths. SCX_TASK_IN_CUSTODY is cleared under the lock serializing the transfer so that the callback is invoked exactly once. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Acked-by: Andrea Righi Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 44 ++++++++++++++++++------------------- kernel/sched/ext/internal.h | 3 +-- kernel/sched/ext/sub.c | 10 ++++----- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e207ccd17164..f568fd9973f6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1499,27 +1499,22 @@ static inline bool task_scx_migrating(struct task_struct *p) return p->scx.sticky_cpu >= 0; } -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) +/* Must be called under the lock serializing @p's custody transfers. */ +static bool task_leave_custody(struct task_struct *p) { if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + return false; p->scx.flags &= ~SCX_TASK_IN_CUSTODY; + return true; } static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - call_task_dequeue(sch, rq, p, 0); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* * Only local inserts get the wakeup treatment below. Rejects kick the @@ -1705,20 +1700,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, if (is_rq_owned) { rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { + bool call_dequeue = false; + /* * Global and bypass DSQs are terminal - the task leaves the - * scheduler's custody, so ops.dequeue() fires here. It can run + * scheduler's custody, so ops.dequeue() fires. It can run * without @p's rq lock (finish_dispatch() passes the dispatch * rq); that's safe because dequeue_task_scx() waits on * SCX_OPSS_DISPATCHING (see the ops_state note above) and so * can't race it. A non-terminal DSQ keeps the task in custody. + * The custody transfer happens under @dsq->lock so that later + * consumers see the flag clear; the callback runs after + * @dsq->lock is dropped because it may lock a DSQ itself. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); + call_dequeue = task_leave_custody(p); else p->scx.flags |= SCX_TASK_IN_CUSTODY; raw_spin_unlock(&dsq->lock); + + if (call_dequeue && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); } /* @@ -2215,7 +2218,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the + * CPU has already passed task_leave_custody() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, @@ -2263,7 +2266,8 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ - call_task_dequeue(sch, rq, p, deq_flags); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) @@ -2379,14 +2383,10 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl } void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) + u64 enq_flags, struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ - if (!dsq_is_rq_owned(src_dsq)) - lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -2634,8 +2634,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { scx_task_unlink_from_dsq(p, src_dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); @@ -2685,8 +2685,8 @@ bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, if (rq == task_rq) { scx_task_unlink_from_dsq(p, dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 115c96fbf932..d150de10a5c9 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2065,8 +2065,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq); + u64 enq_flags, struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a30c965b175d..3036190b0662 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -555,8 +555,8 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx.dsq_list.node); scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); scx_rescue_admit(rq, p, slice); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { @@ -572,7 +572,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, - &rq->scx.local_dsq, rq); + rq); } out_arm: scx_rescue_timer_arm(rq); @@ -596,8 +596,8 @@ void scx_rescue_flush(struct rq *rq) /* and flush out all pending ones */ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); } timer_delete(&rq->scx.rescue.timer); From 9ec7ba20c97d92fa59b351832aeeb934b3b16177 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Thu, 17 Sep 2026 15:52:42 +0800 Subject: [PATCH 4/7] selftests/sched_ext: Test that ops.dequeue() can iterate the consumed DSQ Add a scheduler whose ops.dequeue() iterates the source user DSQ with bpf_iter_scx_dsq. The iteration takes the DSQ's raw spinlock; on a kernel that runs ops.dequeue() while the consume path still holds that lock, the first task consumed self-deadlocks the CPU with IRQs disabled. The watchdog cannot recover from that state, so on an unfixed kernel this test wedges the system instead of failing cleanly. On a fixed kernel the scheduler runs clean and the test passes. Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + .../selftests/sched_ext/dequeue_iter.bpf.c | 73 +++++++++++++++++ .../selftests/sched_ext/dequeue_iter.c | 79 +++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/dequeue_iter.bpf.c create mode 100644 tools/testing/selftests/sched_ext/dequeue_iter.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 3cfe90e0f34f..49897727f535 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -164,6 +164,7 @@ all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubs auto-test-targets := \ create_dsq \ dequeue \ + dequeue_iter \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c new file mode 100644 index 000000000000..76c2c71a90b6 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ops.dequeue() of this scheduler iterates the user DSQ it consumes + * tasks from with bpf_iter_scx_dsq, which takes the DSQ lock. + * On a kernel that still runs ops.dequeue() with that lock held, the + * iteration self-deadlocks the CPU - this test wedges the system on + * unfixed kernels instead of failing cleanly. + * + * Copyright (c) 2026 fangqiurong + */ + +#include + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +#define TEST_DSQ_ID 1000 + +u64 dq_count; + +s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_iter_init) +{ + return scx_bpf_create_dsq(TEST_DSQ_ID, -1); +} + +s32 BPF_STRUCT_OPS(dequeue_iter_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(dequeue_iter_enqueue, struct task_struct *p, u64 enq_flags) +{ + scx_bpf_dsq_insert(p, TEST_DSQ_ID, SCX_SLICE_DFL, enq_flags); +} + +void BPF_STRUCT_OPS(dequeue_iter_dispatch, s32 cpu, struct task_struct *task) +{ + scx_bpf_dsq_move_to_local(TEST_DSQ_ID, 0); +} + +void BPF_STRUCT_OPS(dequeue_iter_dequeue, struct task_struct *p, u64 deq_flags) +{ + struct bpf_iter_scx_dsq it; + struct task_struct *t; + + if (!bpf_iter_scx_dsq_new(&it, TEST_DSQ_ID, 0)) { + while ((t = bpf_iter_scx_dsq_next(&it))) + ; + } + bpf_iter_scx_dsq_destroy(&it); + + __sync_fetch_and_add(&dq_count, 1); +} + +void BPF_STRUCT_OPS(dequeue_iter_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); + scx_bpf_destroy_dsq(TEST_DSQ_ID); +} + +SEC(".struct_ops.link") +struct sched_ext_ops dequeue_iter_ops = { + .init = (void *)dequeue_iter_init, + .select_cpu = (void *)dequeue_iter_select_cpu, + .enqueue = (void *)dequeue_iter_enqueue, + .dispatch = (void *)dequeue_iter_dispatch, + .dequeue = (void *)dequeue_iter_dequeue, + .exit = (void *)dequeue_iter_exit, + .timeout_ms = 1000U, + .name = "dequeue_iter", +}; diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.c b/tools/testing/selftests/sched_ext/dequeue_iter.c new file mode 100644 index 000000000000..f60711bf3b39 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 fangqiurong + */ +#include +#include +#include +#include +#include "dequeue_iter.bpf.skel.h" +#include "scx_test.h" + +#define DQ_TARGET 10 +#define DQ_DEADLINE_MS 3000 + +static unsigned long long now_ms(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return ts.tv_sec * 1000ULL + ts.tv_nsec / 1000000; +} + +static enum scx_test_status setup(void **ctx) +{ + struct dequeue_iter *skel; + + skel = dequeue_iter__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(dequeue_iter__load(skel), "Failed to load skel"); + + *ctx = skel; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + struct dequeue_iter *skel = ctx; + struct bpf_link *link; + unsigned long long end; + + link = bpf_map__attach_struct_ops(skel->maps.dequeue_iter_ops); + SCX_FAIL_IF(!link, "Failed to attach scheduler"); + + end = now_ms() + DQ_DEADLINE_MS; + while (skel->bss->dq_count < DQ_TARGET && !UEI_EXITED(skel, uei) && + now_ms() < end) + usleep(100); + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + + if (skel->bss->dq_count < DQ_TARGET) { + SCX_ERR("ops.dequeue() fired only %llu times", + (unsigned long long)skel->bss->dq_count); + return SCX_TEST_FAIL; + } + + return SCX_TEST_PASS; +} + +static void cleanup(void *ctx) +{ + struct dequeue_iter *skel = ctx; + + dequeue_iter__destroy(skel); +} + +struct scx_test dequeue_iter = { + .name = "dequeue_iter", + .description = "Verify ops.dequeue() can iterate its source user DSQ", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; +REGISTER_SCX_TEST(&dequeue_iter) From 3bd46666cfe51e2bd333fb46a0234694ca594230 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 18 Sep 2026 14:28:37 -1000 Subject: [PATCH 5/7] sched_ext: Pass the initial cmask to cid-form ops.enable() The cid-form API has an obvious hole. A task's cid mask is only visible through ops.set_cmask(), which fires on affinity changes and class switches but not when a task enters a scheduler through fork, sub-sched enable or re-home, and there is no p->cpus_ptr equivalent to fall back on. Schedulers work around it by seeding the mask in ops.init_task() from p->cpus_ptr cid by cid, which is subtly wrong: on sub-sched enable and re-home, an affinity change between init_task() and enable() is delivered to the sched the task is still on, and nothing corrects the new sched's copy afterwards. Fix it by adding struct scx_enable_args to cid-form ops.enable() carrying the task's cmask, built in the per-cpu scratch under the rq lock as the task enters the scheduler, and calling set_cmask() with the same mask right after enable(), ahead of set_weight(). A scheduler can then track affinity in set_cmask() alone, and scx_qmap drops its init_task() seed. set_cmask() no longer fires for a cid-form task before it is enabled, and the class-switch republish in switching_to_scx() is limited to the cpu form. This changes the cid-form ops.enable() signature, which is fine as the cid-form API is still considered unreleased. An args struct rather than a bare cmask argument leaves room for more initial state without another signature change, and the cmask travels as a plain arena address because BTF can't type arena struct members yet. v2: The cmask travels as a u64 arena address, cmask_arena_addr, instead of a kernel-typed pointer, with the typing limitation and the planned typed alias documented (Sashiko review). v3: The initial set_cmask() is delivered before set_weight() so that the mask is in place when weight-dependent state is derived (Andrea Righi). Selftest added. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 92 +++++++++++++++++++++++----------- kernel/sched/ext/internal.h | 56 ++++++++++++++++++--- tools/sched_ext/scx_qmap.bpf.c | 3 -- 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f568fd9973f6..ad391a8cbd05 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -447,37 +447,45 @@ static void switch_rq_lock(struct rq *from, struct rq *to) DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); /** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask + * @sch: scx_sched whose scratch to fill + * @cpumask: cpus to translate into cids + * + * The scratch lives in BPF-writable arena memory and its header can't be + * trusted, so it is rewritten from kernel geometry rather than read. Caller + * must hold an rq lock so this cpu is the sole kernel writer for as long as the + * returned address is in use. + */ +static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, + const struct cpumask *cpumask) +{ + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + return kern_va; +} + +/** + * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch + * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - struct scx_cmask_ref ref; - - /* - * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header. set_cmask()'s __arena - * argument takes the kernel address and the struct_ops - * trampoline rebases it into BPF's arena pointer form. The rq - * lock makes this cpu the sole kernel writer. - */ - scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); - scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); - } else { + if (scx_is_cid_type()) + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_fill_cmask_scratch(sch, cpumask)); + else SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); - } } enum scx_dsq_iter_flags { @@ -3634,8 +3642,12 @@ static void set_cpus_allowed_scx(struct task_struct *p, * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. + * + * The cid form receives the initial mask when the task is enabled and + * hears about changes only afterwards, see struct scx_enable_args. */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (SCX_HAS_OP(sch, set_cpumask) && + (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3944,8 +3956,27 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) p->scx.weight = sched_weight_to_cgroup(weight); - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); + if (SCX_HAS_OP(sch, enable)) { + if (scx_is_cid_type()) { + struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); + struct scx_enable_args args = { + .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), + }; + + SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); + } else { + SCX_CALL_OP_TASK(sch, enable, rq, p); + } + } + + /* + * The initial mask also goes out through set_cmask() so a scheduler can + * track affinity there alone, and before set_weight() so that the mask + * is in place when weight-dependent state is derived, see struct + * scx_enable_args. + */ + if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); @@ -4288,9 +4319,10 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) /* * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. + * different scheduler class. Keep the BPF scheduler up-to-date. The cid + * form gets its mask from scx_enable_task(). */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } @@ -8374,10 +8406,11 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused. Some need - * fresh stubs, set_cmask due to an argument type difference and the sub-sched - * notifiers because no cpu-form stub exists to reuse. + * fresh stubs, set_cmask and enable due to argument differences and the + * sub-sched notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} @@ -8398,7 +8431,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, + .enable = sched_ext_ops_cid__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cpuctl_init = sched_ext_ops__cgroup_init, @@ -10421,8 +10454,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) if (unlikely(!online)) return NULL; - /* BPF rebases by the low 32 bits, like __arena callback args */ - return (void *)((unsigned long)online - sch->arena_kern_base); + return (void *)scx_kaddr_to_arena(sch, online); } /** diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d150de10a5c9..1df8f583b0ec 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -250,6 +250,31 @@ struct scx_exit_task_args { bool cancelled; }; +/** + * struct scx_enable_args - Argument container for cid-form ops.enable() + * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on + * + * @cmask_arena_addr is the task's affinity as it enters the scheduler. + * set_cmask() delivers the same mask right after enable(), before set_weight() + * and the first enqueue, then every affinity change afterwards, and is never + * called before enable(). A scheduler may therefore track affinity in + * set_cmask() alone. + * + * The kernel builds the mask in the scheduler arena from its own geometry, so + * the header is valid regardless of what the scheduler last wrote there. The + * memory is per-cpu scratch reused once the callback returns: copy the bits + * out, don't keep the address. The set_cmask() argument follows the same rules. + * + * The address is a plain value rather than a typed pointer because BTF can't + * mark a struct member as an arena pointer yet and a pointer member would reach + * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena * + * before use. Once arena members can be typed, a typed alias will join this + * field in an anonymous union at the same offset. + */ +struct scx_enable_args { + u64 cmask_arena_addr; +}; + /* argument container for ops.cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ @@ -1037,6 +1062,7 @@ struct sched_ext_ops { * - dispatch -> dispatch (cpu arg is now cid) * - update_idle -> update_idle (cpu arg is now cid) * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - enable -> enable (takes struct scx_enable_args) * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid @@ -1070,7 +1096,7 @@ struct sched_ext_ops_cid { struct scx_init_task_args *args); void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); + void (*enable)(struct task_struct *p, struct scx_enable_args *args); void (*disable)(struct task_struct *p); void (*dump)(struct scx_dump_ctx *ctx); void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); @@ -1533,7 +1559,8 @@ struct scx_sched { * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel * access either view of the same storage without function-pointer * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + * callbacks whose cid-form signature differs (set_cmask, enable, + * select_cid, cid_online, ...). */ union { struct sched_ext_ops ops; @@ -1556,9 +1583,9 @@ struct scx_sched { uintptr_t arena_kern_base; /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and passes it to the callback's __arena argument. + * Per-CPU arena cmask the kernel fills from a task's cpumask and hands + * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are + * the kernel addresses. */ struct scx_cmask * __percpu *set_cmask_scratch; struct scx_cmask *online_cmask; @@ -1669,6 +1696,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } +/** + * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel address inside @sch's arena + * + * __arena callback arguments need no translation. Addresses handed to BPF any + * other way, such as struct fields and kfunc return values, go through this. + */ +static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (uintptr_t)kaddr - sch->arena_kern_base; +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -2302,9 +2342,9 @@ do { \ } while (0) /* - * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs - * this: it takes an arena cmask address instead of a cpumask, so it cannot be - * invoked via its cpu-form set_cpumask() slot. + * Dispatch a task op through the cid-form ops_cid table, for the ops whose + * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena + * cmask instead of a cpumask and enable() takes scx_enable_args. */ #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 67b7c01cae55..2f3653199842 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -961,9 +961,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_task, struct task_struct *p, taskc->highpri = false; taskc->core_sched_seq = 0; cmask_init(&taskc->cpus_allowed, 0, scx_bpf_nr_cids()); - bpf_rcu_read_lock(); - cmask_from_cpumask(&taskc->cpus_allowed, p->cpus_ptr); - bpf_rcu_read_unlock(); v = bpf_task_storage_get(&task_ctx_stor, p, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE); From d781d1b78acf547bafeb1592e8ba4020dce515c6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 19 Sep 2026 04:08:20 -1000 Subject: [PATCH 6/7] selftests/sched_ext: Check the cmask cid-form ops.enable() receives cid-form ops.enable() now hands the task's cmask to the scheduler and set_cmask() repeats it right after. Add a cid-form selftest that checks both against p->cpus_ptr, that they match each other, that the initial set_cmask() lands before set_weight() and before the task first becomes runnable, and that set_cmask() never precedes enable(), across class-switch enables, fork-path enables and live affinity changes. v2: Mismatch details returned through a caller-local struct instead of globals, alloc_words validated in the header check, loop bounded by nr_cids directly (Andrea Righi). Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + .../selftests/sched_ext/enable_cmask.bpf.c | 217 ++++++++++++++++++ .../selftests/sched_ext/enable_cmask.c | 138 +++++++++++ 3 files changed, 356 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/enable_cmask.bpf.c create mode 100644 tools/testing/selftests/sched_ext/enable_cmask.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 49897727f535..4e06d0baaeec 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -169,6 +169,7 @@ auto-test-targets := \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ dsp_local_on \ + enable_cmask \ enq_select_cpu \ exit \ hotplug \ diff --git a/tools/testing/selftests/sched_ext/enable_cmask.bpf.c b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c new file mode 100644 index 000000000000..0068f3c7ab3c --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A cid-form scheduler checking the cmask cid-form ops.enable() receives: the + * header, every cid bit against p->cpus_ptr, and that set_cmask() follows with + * the same mask before set_weight() and before the task first becomes runnable, + * and never runs before enable(). + * + * Copyright (c) 2026 Tejun Heo + */ +#include + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1 << 16); +} arena SEC(".maps"); + +struct task_ctx { + u64 enable_fp; /* fingerprint of the mask enable() received */ + bool enabled; + bool pending; /* enable() ran, the initial set_cmask() hasn't */ +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +/* details of a cid bit mismatch, filled by check_mask() */ +struct mask_mismatch { + s32 cid; + bool want; + bool got; +}; + +u64 nr_enable, nr_initial_set_cmask, nr_set_cmask, nr_set_weight; + +UEI_DEFINE(uei); + +static struct task_ctx *lookup_task_ctx(struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) + scx_bpf_error("task_ctx lookup failed for %s[%d]", p->comm, p->pid); + return tctx; +} + +/* + * Verify @m's header and every cid bit against @p's cpumask and fingerprint the + * bits into @fp. Return 0 on success, -EINVAL on a bad header, -ENOENT on a cid + * without a cpu and -EIO on a bit mismatch with the details in @mm. + */ +static int check_mask(struct task_struct *p, const struct scx_cmask __arena *m, u64 *fp, + struct mask_mismatch *mm) +{ + u32 nr_cids = scx_bpf_nr_cids(); + u64 h = 0; + s32 cid; + + if (m->base || m->nr_cids != nr_cids || m->alloc_words != CMASK_NR_WORDS(nr_cids)) + return -EINVAL; + + bpf_for(cid, 0, nr_cids) { + bool want, got; + s32 cpu; + + cpu = scx_bpf_cid_to_cpu(cid); + if (cpu < 0) + return -ENOENT; + want = bpf_cpumask_test_cpu(cpu, p->cpus_ptr); + got = cmask_test(cid, m); + if (want != got) { + mm->cid = cid; + mm->want = want; + mm->got = got; + return -EIO; + } + h = h * 31 + got; + } + + *fp = h; + return 0; +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enable_cmask_init_task, struct task_struct *p, + struct scx_init_task_args *args) +{ + if (!bpf_task_storage_get(&task_ctx_stor, p, 0, BPF_LOCAL_STORAGE_GET_F_CREATE)) + return -ENOMEM; + return 0; +} + +void BPF_STRUCT_OPS(enable_cmask_enable, struct task_struct *p, struct scx_enable_args *args) +{ + struct scx_cmask __arena *m = (struct scx_cmask __arena *)args->cmask_arena_addr; + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_enable, 1); + if (tctx->enabled || tctx->pending) { + scx_bpf_error("enable: %s[%d] enabled twice", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &tctx->enable_fp, &mm); + if (ret) { + scx_bpf_error("enable: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + tctx->enabled = true; + tctx->pending = true; +} + +void BPF_STRUCT_OPS(enable_cmask_set_cmask, struct task_struct *p, + struct scx_cmask __arena *m) +{ + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + u64 fp; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_cmask, 1); + if (!tctx->enabled) { + scx_bpf_error("set_cmask: %s[%d] not enabled", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &fp, &mm); + if (ret) { + scx_bpf_error("set_cmask: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + + if (tctx->pending) { + if (fp != tctx->enable_fp) { + scx_bpf_error("set_cmask: %s[%d] initial mask differs from enable()", + p->comm, p->pid); + return; + } + tctx->pending = false; + __sync_fetch_and_add(&nr_initial_set_cmask, 1); + } +} + +void BPF_STRUCT_OPS(enable_cmask_set_weight, struct task_struct *p, u32 weight) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_weight, 1); + if (tctx->pending) + scx_bpf_error("set_weight: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_runnable, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + if (tctx->pending) + scx_bpf_error("runnable: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_disable, struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + tctx->enabled = false; + tctx->pending = false; +} + +void BPF_STRUCT_OPS(enable_cmask_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SCX_OPS_CID_DEFINE(enable_cmask_ops, + .init_task = (void *)enable_cmask_init_task, + .enable = (void *)enable_cmask_enable, + .set_cmask = (void *)enable_cmask_set_cmask, + .set_weight = (void *)enable_cmask_set_weight, + .runnable = (void *)enable_cmask_runnable, + .disable = (void *)enable_cmask_disable, + .exit = (void *)enable_cmask_exit, + .flags = SCX_OPS_SWITCH_PARTIAL, + .name = "enable_cmask"); diff --git a/tools/testing/selftests/sched_ext/enable_cmask.c b/tools/testing/selftests/sched_ext/enable_cmask.c new file mode 100644 index 000000000000..556bcad4431d --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Tejun Heo */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "enable_cmask.bpf.skel.h" +#include "scx_test.h" + +#define SCHED_EXT 7 +#define NR_CHILDREN 8 +#define MAX_CPUS 1024 + +static int cpus[MAX_CPUS]; +static int nr_cpus; + +static void spin_ms(int ms) +{ + struct timespec start, now; + + clock_gettime(CLOCK_MONOTONIC, &start); + do { + clock_gettime(CLOCK_MONOTONIC, &now); + } while ((now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000 < ms); +} + +static int pin(pid_t pid, int idx) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpus[idx % nr_cpus], &set); + return sched_setaffinity(pid, sizeof(set), &set); +} + +/* + * Pin, switch to SCHED_EXT for a class-switch enable, fork a grandchild that + * inherits the policy for a fork-path enable, then change affinity a few times + * while running for set_cmask() on live tasks. + */ +static int child(int idx) +{ + struct sched_param param = {}; + int i, status; + pid_t pid; + + if (pin(0, idx) || sched_setscheduler(0, SCHED_EXT, ¶m)) + return 1; + + pid = fork(); + if (pid < 0) + return 1; + if (!pid) { + spin_ms(20); + return 0; + } + + for (i = 1; i <= 4; i++) { + if (pin(0, idx + i)) + return 1; + spin_ms(5); + } + + return waitpid(pid, &status, 0) == pid && !status ? 0 : 1; +} + +static enum scx_test_status run(void *ctx) +{ + struct enable_cmask *skel; + struct bpf_link *link; + pid_t pids[NR_CHILDREN]; + cpu_set_t set; + int i, status, failed = 0; + + if (!__COMPAT_struct_has_field("scx_enable_args", "cmask_arena_addr")) + return SCX_TEST_SKIP; + + SCX_FAIL_IF(sched_getaffinity(0, sizeof(set), &set), "Failed to read affinity"); + for (i = 0; i < MAX_CPUS && i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &set)) + cpus[nr_cpus++] = i; + if (nr_cpus < 2) + return SCX_TEST_SKIP; + + skel = enable_cmask__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(enable_cmask__load(skel), "Failed to load skel"); + + link = bpf_map__attach_struct_ops(skel->maps.enable_cmask_ops); + SCX_FAIL_IF(!link, "Failed to attach struct_ops"); + + for (i = 0; i < NR_CHILDREN; i++) { + pids[i] = fork(); + SCX_FAIL_IF(pids[i] < 0, "Failed to fork"); + if (!pids[i]) + exit(child(i)); + } + + /* affinity changes from the outside race with the children's own */ + for (i = 0; i < NR_CHILDREN; i++) + pin(pids[i], i + NR_CHILDREN); + + for (i = 0; i < NR_CHILDREN; i++) { + if (waitpid(pids[i], &status, 0) != pids[i] || status) + failed++; + } + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + SCX_EQ(failed, 0); + SCX_GE(skel->bss->nr_enable, 2 * NR_CHILDREN); + SCX_EQ(skel->bss->nr_initial_set_cmask, skel->bss->nr_enable); + SCX_GT(skel->bss->nr_set_cmask, skel->bss->nr_initial_set_cmask); + SCX_GE(skel->bss->nr_set_weight, skel->bss->nr_enable); + printf("enable=%lu initial_set_cmask=%lu set_cmask=%lu set_weight=%lu\n", + (unsigned long)skel->bss->nr_enable, + (unsigned long)skel->bss->nr_initial_set_cmask, + (unsigned long)skel->bss->nr_set_cmask, + (unsigned long)skel->bss->nr_set_weight); + + enable_cmask__destroy(skel); + return SCX_TEST_PASS; +} + +struct scx_test enable_cmask = { + .name = "enable_cmask", + .description = "Check the cid-form ops.enable() cmask and the set_cmask() after it", + .run = run, +}; +REGISTER_SCX_TEST(&enable_cmask) From 4409a85735cdca5c4c400c0dad1feee091872eee Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Wed, 23 Sep 2026 19:07:37 +0800 Subject: [PATCH 7/7] sched_ext: Count SCX_EV_SUB_BYPASS_DISPATCH in the dispatch fallback When a descendant scheduler enters bypass mode, its tasks are parked in the bypass DSQs of the nearest non-bypassing ancestor, which is then responsible for running them. On behalf of such a non-bypassing host, scx_dispatch_sched() consumes those bypass DSQs from two places: the attempt made every SCX_BYPASS_HOST_NTH dispatches, and the end-of-dispatch fallback that keeps the CPU from going idle while bypassed descendants still have tasks queued. The former increments SCX_EV_SUB_BYPASS_DISPATCH but the latter does not, even though both perform the same scx_consume_dispatch_q() on the same bypass DSQ. The descendant bypass dispatches done by the fallback are therefore missing from the counter exposed via sysfs, scx_dump_state() and the scx_bpf_events() kfunc, which under-reports the actual number of such dispatches. Add the missing __scx_add_event() so the fallback counts them too. When @sch itself is bypassing, scx_dispatch_sched() takes the earlier self-bypass branch and returns before reaching these host paths; that mode is accounted for by SCX_EV_BYPASS_DISPATCH at enqueue time and is intentionally left unchanged. Fixes: 025b1bd41965 ("sched_ext: Implement hierarchical bypass mode") Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/inlines.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index ed423bcc26b8..2ff5479334cb 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -129,8 +129,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; + } return SCX_DSP_NONE; }