sched_ext: Fixes for v7.2-rc3

- Lifecycle fixes for the new sub-scheduler support: two use-after-frees
   and an enable-failure path that left a half-initialized sub-scheduler
   linked.
 
 - Two dispatch-path locking bugs: a spurious scheduler abort from a
   migration race, and a lockdep splat from stale runqueue-lock tracking.
 
 - Callback and task-state fixes: stale scheduler-owned state on a task
   leaving SCX, a weight callback running after disable, and a bogus
   warning on core-scheduling forced idle.
 
 - On nohz_full, finite-slice tasks could miss the tick that expires their
   slice. Enable it when such a task is picked, with a selftest.
 
 - Smaller fixes: userspace CPU-mask helpers, ratelimited deprecation
   warnings, docs and a sparse annotation.
 -----BEGIN PGP SIGNATURE-----
 
 iIQEABYKACwWIQTfIjM1kS57o3GsC/uxYfJx3gVYGQUCalVWgw4cdGpAa2VybmVs
 Lm9yZwAKCRCxYfJx3gVYGTmUAP0QHX7Ae/g7yMPEB4slURpnSj/wE7hRCI5jTmay
 2iIEJgD/RAhpwUUAp4Abozt0mjQMWh9UyEVjOxPCVNKaNh1XfwU=
 =EBCm
 -----END PGP SIGNATURE-----

Merge tag 'sched_ext-for-7.2-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Pull sched_ext fixes from Tejun Heo:

 - Lifecycle fixes for the new sub-scheduler support: two
   use-after-frees and an enable-failure path that left a
   half-initialized sub-scheduler linked.

 - Two dispatch-path locking bugs: a spurious scheduler abort from a
   migration race, and a lockdep splat from stale runqueue-lock
   tracking.

 - Callback and task-state fixes: stale scheduler-owned state on a task
   leaving SCX, a weight callback running after disable, and a bogus
   warning on core-scheduling forced idle.

 - On nohz_full, finite-slice tasks could miss the tick that expires
   their slice. Enable it when such a task is picked, with a selftest.

 - Smaller fixes: userspace CPU-mask helpers, ratelimited deprecation
   warnings, docs and a sparse annotation.

* tag 'sched_ext-for-7.2-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext:
  sched_ext: Skip ops.set_weight() for disabled tasks
  tools/sched_ext: scx - Fix cmask_subset(), cmask_equal() and cmask_weight()
  sched_ext: Fix premature ops->priv publication in scx_alloc_and_add_sched()
  sched_ext: Record an error on errno-only sub-enable failure
  selftests/sched_ext: Verify nohz_full tick behavior
  sched_ext: Enable tick for finite slices on nohz_full
  sched_ext: Preserve rq tracking across local DSQ dispatch
  sched_ext: Documentation: Fix ops table header reference
  sched_ext: Don't warn on core-sched forced idle in put_prev_task_scx()
  sched_ext: Pin parent scx_sched across a child sub-scheduler's lifetime
  sched_ext: Annotate ksyncs with __rcu in alloc/free_kick_syncs()
  sched_ext: Check remote rq eligibility under task's rq lock
  sched_ext: Reset dsq_vtime and slice when a task leaves SCX
  sched_ext: Avoid flooding the log with deprecation warnings
This commit is contained in:
Linus Torvalds 2026-07-13 15:55:17 -07:00
commit f7574d3f90
7 changed files with 614 additions and 95 deletions

View File

@ -493,8 +493,9 @@ a freshly woken up task gets on a CPU.
Where to Look
=============
* ``include/linux/sched/ext.h`` defines the core data structures, ops table
and constants.
* ``include/linux/sched/ext.h`` defines the core data structures and
constants, while the ops table (``struct sched_ext_ops``) is defined in
``kernel/sched/ext/internal.h``.
* ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers.
The functions prefixed with ``scx_bpf_`` can be called from the BPF
@ -555,7 +556,8 @@ ABI Instability
===============
The APIs provided by sched_ext to BPF schedulers programs have no stability
guarantees. This includes the ops table callbacks and constants defined in
guarantees. This includes the ops table callbacks defined in
``kernel/sched/ext/internal.h`` and the constants defined in
``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``.

View File

@ -479,6 +479,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
*/
DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
static void switch_rq_lock(struct rq *from, struct rq *to)
{
bool tracked = scx_locked_rq() == from;
if (tracked)
update_locked_rq(NULL);
raw_spin_rq_unlock(from);
raw_spin_rq_lock(to);
if (tracked)
update_locked_rq(to);
}
/*
* Flipped on enable per sch->is_cid_type. Declared in internal.h so
* subsystem inlines can read it.
@ -2274,8 +2286,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
deactivate_task(src_rq, p, 0);
set_task_cpu(p, cpu_of(dst_rq));
raw_spin_rq_unlock(src_rq);
raw_spin_rq_lock(dst_rq);
switch_rq_lock(src_rq, dst_rq);
/*
* We want to pass scx-specific enq_flags but activate_task() will
@ -2307,6 +2318,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
* no to the BPF scheduler initiated migrations while offline.
*
* The caller must ensure that @p and @rq are on different CPUs.
* If enforce == true, caller must hold @p's rq lock.
*/
static bool task_can_run_on_remote_rq(struct scx_sched *sch,
struct task_struct *p, struct rq *rq,
@ -2314,6 +2326,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
{
s32 cpu = cpu_of(rq);
/*
* To prevent races with @p still running on its old CPU while switching
* out, make sure we're holding @p's rq lock so as not to risk
* erroneously killing the BPF scheduler.
*/
if (enforce)
lockdep_assert_rq_held(task_rq(p));
WARN_ON_ONCE(task_cpu(p) == cpu);
/*
@ -2581,13 +2601,6 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
return;
}
if (src_rq != dst_rq &&
unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p,
enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK);
return;
}
/*
* @p is on a possibly remote @src_rq which we need to lock to move the
* task. If dequeue is in progress, it'd be locking @src_rq and waiting
@ -2606,14 +2619,14 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
/* switch to @src_rq lock */
if (locked_rq != src_rq) {
raw_spin_rq_unlock(locked_rq);
switch_rq_lock(locked_rq, src_rq);
locked_rq = src_rq;
raw_spin_rq_lock(src_rq);
}
/* task_rq couldn't have changed if we're still the holding cpu */
if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
!WARN_ON_ONCE(src_rq != task_rq(p))) {
bool fallback = false;
/*
* If @p is staying on the same rq, there's no need to go
* through the full deactivate/activate cycle. Optimize by
@ -2623,6 +2636,11 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
p->scx.holding_cpu = -1;
dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
enq_flags);
} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
p->scx.holding_cpu = -1;
fallback = true;
dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
} else {
move_remote_task_to_local_dsq(p, enq_flags,
src_rq, dst_rq);
@ -2631,15 +2649,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
}
/* if the destination CPU is idle, wake it up */
if (sched_class_above(p->sched_class, dst_rq->curr->sched_class))
if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
resched_curr(dst_rq);
}
/* switch back to @rq lock */
if (locked_rq != rq) {
raw_spin_rq_unlock(locked_rq);
raw_spin_rq_lock(rq);
}
if (locked_rq != rq)
switch_rq_lock(locked_rq, rq);
}
/**
@ -2970,24 +2986,38 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
/*
* @p is getting newly scheduled or got kicked after someone updated its
* slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
* slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
* stopped. See scx_can_stop_tick().
*
* Moreover, refresh the load_avgs just when transitioning in and out of
* nohz. In the future, we might want to add a mechanism to update
* load_avgs periodically on tick-stopped CPUs.
*/
if ((p->scx.slice == SCX_SLICE_INF) !=
(bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
if (p->scx.slice == SCX_SLICE_INF)
if (p->scx.slice == SCX_SLICE_INF) {
if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
/*
* Bypass mode always assigns finite slices, so @p
* can't have an infinite slice while bypassing.
* Therefore, sched_update_tick_dependency() can safely
* evaluate the outgoing task.
*/
rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
else
rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
sched_update_tick_dependency(rq);
sched_update_tick_dependency(rq);
update_other_load_avgs(rq);
}
} else {
if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
update_other_load_avgs(rq);
}
/*
* For now, let's refresh the load_avgs just when transitioning
* in and out of nohz. In the future, we might want to add a
* mechanism which calls the following periodically on
* tick-stopped CPUs.
* @rq still references the outgoing scheduling context. A finite
* slice is sufficient by itself to require the tick.
*/
update_other_load_avgs(rq);
if (tick_nohz_full_cpu(cpu_of(rq)))
tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
}
}
@ -3082,9 +3112,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
* sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
* ops.enqueue() that @p is the only one available for this cpu,
* which should trigger an explicit follow-up scheduling event.
*
* Core scheduling can force this CPU idle while @p stays
* runnable. @p's cookie then won't match the core's, so skip
* the warning in that case.
*/
if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) &&
!(sch->ops.flags & SCX_OPS_ENQ_LAST));
do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
} else {
do_enqueue_task(rq, p, 0, -1);
@ -3647,6 +3682,13 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
SCX_CALL_OP_TASK(sch, disable, rq, p);
scx_set_task_state(p, SCX_TASK_READY);
/*
* Reset the SCX-managed fields when @p leaves the BPF scheduler's
* control, after ops.disable() has observed their final values.
*/
p->scx.dsq_vtime = 0;
p->scx.slice = 0;
/*
* Verify the task is not in BPF scheduler's custody. If flag
* transitions are consistent, the flag should always be clear
@ -3925,6 +3967,17 @@ static void reweight_task_scx(struct rq *rq, struct task_struct *p,
if (task_dead_and_done(p))
return;
/*
* When switching sched_class away from SCX, reweight_task_scx()
* is called _after_ scx_disable_task(). Skip calling ops.set_weight()
* since the BPF scheduler may have already forgotten the task in
* ops.disable().
* p->scx.weight will be recalculated in scx_enable_task() if the task
* ever returns to SCX class.
*/
if (scx_get_task_state(p) != SCX_TASK_ENABLED)
return;
p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
if (SCX_HAS_OP(sch, set_weight))
SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@ -4301,6 +4354,15 @@ bool scx_can_stop_tick(struct rq *rq)
if (p->sched_class != &ext_sched_class)
return true;
/*
* @rq->curr may still reference an outgoing EXT task after it has been
* dequeued. If no EXT tasks are accounted on @rq, ignore its stale
* slice state. If another task is dispatched from a DSQ,
* set_next_task_scx() will update the dependency for the incoming task.
*/
if (!rq->scx.nr_running)
return true;
if (scx_bypassing(sch, cpu_of(rq)))
return false;
@ -4901,6 +4963,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
cgroup_put(sch_cgroup(sch));
if (sch->sub_kset)
kobject_put(&sch->sub_kset->kobj);
if (scx_parent(sch))
kobject_put(&scx_parent(sch)->kobj);
#endif /* CONFIG_EXT_SUB_SCHED */
for_each_possible_cpu(cpu) {
@ -5672,7 +5736,7 @@ static void free_kick_syncs(void)
int cpu;
for_each_possible_cpu(cpu) {
struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *to_free;
to_free = rcu_replace_pointer(*ksyncs, NULL, true);
@ -6653,7 +6717,7 @@ static int alloc_kick_syncs(void)
* can exceed percpu allocator limits on large machines.
*/
for_each_possible_cpu(cpu) {
struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *new_ksyncs;
WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
@ -6825,11 +6889,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->ops = *cmd->ops;
}
rcu_assign_pointer(ops->priv, sch);
sch->kobj.kset = scx_kset;
INIT_LIST_HEAD(&sch->all);
#ifdef CONFIG_EXT_SUB_SCHED
char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
if (!buf) {
@ -6847,13 +6906,32 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->cgrp = cgrp;
INIT_LIST_HEAD(&sch->children);
INIT_LIST_HEAD(&sch->sibling);
#endif /* CONFIG_EXT_SUB_SCHED */
if (parent)
/*
* Publishing makes @sch visible to scx_prog_sched() readers. Failure
* paths after this point must free @sch through kobject_put() whose
* release path defers the actual freeing by an RCU grace period.
*/
rcu_assign_pointer(ops->priv, sch);
sch->kobj.kset = scx_kset;
INIT_LIST_HEAD(&sch->all);
#ifdef CONFIG_EXT_SUB_SCHED
if (parent) {
/*
* Pin @parent for @sch's lifetime. The kobject hierarchy pins
* it only via @parent->sub_kset, which is dropped during
* disable. Released in scx_sched_free_rcu_work().
*/
kobject_get(&parent->kobj);
ret = kobject_init_and_add(&sch->kobj, &scx_ktype,
&parent->sub_kset->kobj,
"sub-%llu", cgroup_id(cgrp));
else
} else {
ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
}
if (ret < 0) {
RCU_INIT_POINTER(ops->priv, NULL);
@ -6895,7 +6973,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
#ifdef CONFIG_EXT_SUB_SCHED
err_free_lb_resched:
RCU_INIT_POINTER(ops->priv, NULL);
free_cpumask_var(sch->bypass_lb_resched_cpumask);
#endif
err_free_lb_cpumask:
@ -6988,7 +7065,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
* run past the BPF allocation. Skip for cid-form.
*/
if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
/*
* Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
@ -7686,6 +7763,12 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
percpu_up_write(&scx_fork_rwsem);
err_disable:
mutex_unlock(&scx_enable_mutex);
/*
* Some enable failures only return an errno (e.g. -ENOMEM from an
* allocation) without calling scx_error(). Record it so
* scx_flush_disable_work() runs the disable and ops.exit() fires.
*/
scx_error(sch, "scx_sub_enable() failed (%d)", ret);
scx_flush_disable_work(sch);
cmd->ret = 0;
}
@ -7806,7 +7889,7 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
off + size <= offsetofend(struct task_struct, scx.slice)) ||
(off >= offsetof(struct task_struct, scx.dsq_vtime) &&
off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) {
pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()");
pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n");
return SCALAR_VALUE;
}
@ -8796,10 +8879,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
if (in_balance) {
if (this_rq != src_rq) {
raw_spin_rq_unlock(this_rq);
raw_spin_rq_lock(src_rq);
}
if (this_rq != src_rq)
switch_rq_lock(this_rq, src_rq);
} else {
raw_spin_rq_lock(src_rq);
}
@ -8831,10 +8912,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
dispatched = true;
out:
if (in_balance) {
if (this_rq != locked_rq) {
raw_spin_rq_unlock(locked_rq);
raw_spin_rq_lock(this_rq);
}
if (this_rq != locked_rq)
switch_rq_lock(locked_rq, this_rq);
} else {
raw_spin_rq_unlock_irqrestore(locked_rq, flags);
}

View File

@ -1469,21 +1469,24 @@ static const char *scx_enable_state_str[] = {
* The sched_ext core uses a "lock dancing" protocol coordinated by
* p->scx.holding_cpu. When moving a task to a different rq:
*
* 1. Verify task can be moved (CPU affinity, migration_disabled, etc.)
* 2. Set p->scx.holding_cpu to the current CPU
* 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
* 1. Set p->scx.holding_cpu to the current CPU
* 2. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
* is set, so clearing DISPATCHING first prevents the circular wait
* (safe to lock the rq we need)
* 4. Unlock the current CPU's rq
* 5. Lock src_rq (where the task currently lives)
* 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
* 3. Unlock the current CPU's rq
* 4. Lock src_rq (where the task currently lives)
* 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
* race (dequeue clears holding_cpu to -1 when it takes the task), in
* this case migration is aborted
* 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly
* 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly
* into dst_rq's local DSQ (no lock swap needed)
* 8. Otherwise: call move_remote_task_to_local_dsq(), which releases
* src_rq, locks dst_rq, and performs the deactivate/activate
* migration cycle (dst_rq is held on return)
* 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq
* (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu,
* leave the task on src_rq, and enqueue it on the fallback DSQ.
* 8. Otherwise (i.e. if the task can be moved to dst_rq), call
* move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq,
* and performs the deactivate/activate migration cycle
* (dst_rq is held on return)
* 9. Unlock dst_rq and re-lock the current CPU's rq to restore
* the lock state expected by the caller
*

View File

@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
if (a->base != b->base || a->nr_cids != b->nr_cids)
return false;
nr_words = CMASK_NR_WORDS(a->nr_cids);
if (a->nr_cids == 0)
return true;
nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
bpf_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
return true;
}
/*
* True iff every bit set in @a is also set in @b over the intersection of
* their ranges. Bits of @a outside @b's range fail the test.
*/
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b)
{
u32 a_end = a->base + a->nr_cids;
u32 b_end = b->base + b->nr_cids;
u32 a_wbase = a->base / 64;
u32 b_wbase = b->base / 64;
u32 nr_words, i;
/* any bit of @a outside @b's range is a subset violation */
if (a->base < b->base || a_end > b_end)
return false;
nr_words = CMASK_NR_WORDS(a->nr_cids);
bpf_for(i, 0, CMASK_MAX_WORDS) {
u32 wi_b;
if (i >= nr_words)
break;
wi_b = a_wbase + i - b_wbase;
if (a->bits[i] & ~b->bits[wi_b])
return false;
}
return true;
}
/**
* cmask_next_set - find the first set bit at or after @cid
* @m: cmask to search
@ -488,16 +460,66 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
(cid) < (m)->base + (m)->nr_cids; \
(cid) = cmask_next_set((m), (cid) + 1))
/*
* True iff every bit set in @a is also set in @b. Matches the kernel-side
* scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside
* @b's range fail the test.
*/
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b)
{
u32 a_end = a->base + a->nr_cids;
u32 b_end = b->base + b->nr_cids;
u32 a_wbase = a->base / 64;
u32 b_wbase = b->base / 64;
u32 lo = a->base > b->base ? a->base : b->base;
u32 hi = a_end < b_end ? a_end : b_end;
u32 lo_word, hi_word, i;
/* set bits of @a outside @b's range can't be in @b */
if (a->base < b->base &&
cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end))
return false;
if (a_end > b_end &&
cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end)
return false;
if (lo >= hi)
return true;
/*
* Walk the words the range intersection spans. Plain word tests
* suffice: the scans above guarantee @a has no set bit outside @b's
* range and padding bits are kept clear by all cmask helpers.
*/
lo_word = lo / 64;
hi_word = (hi - 1) / 64;
bpf_for(i, 0, CMASK_MAX_WORDS) {
u32 w = lo_word + i;
if (w > hi_word)
break;
if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase])
return false;
}
return true;
}
/*
* Population count over [base, base + nr_cids). Padding bits in the head/tail
* words are guaranteed zero by the mutating helpers, so a flat popcount over
* all words is correct.
* the words the range spans is correct.
*/
static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
{
u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
u32 nr_words, i;
u32 count = 0;
if (!m->nr_cids)
return 0;
nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
bpf_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;

View File

@ -176,6 +176,7 @@ auto-test-targets := \
maybe_null \
minimal \
non_scx_kfunc_deny \
nohz_tick \
numa \
allowed_cpus \
peek_dsq \

View File

@ -0,0 +1,65 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*
* Exercise tick dependency transitions between infinite and finite slices.
*/
#include <scx/common.bpf.h>
char _license[] SEC("license") = "GPL";
const volatile s32 test_cpu;
bool finite_phase;
u64 nr_inf_running;
u64 nr_finite_running;
u64 nr_finite_ticks;
UEI_DEFINE(uei);
s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu,
u64 wake_flags)
{
return prev_cpu;
}
void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags)
{
u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF;
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags);
if (enq_flags & SCX_ENQ_LAST)
scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE);
}
void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p)
{
if (bpf_get_smp_processor_id() != test_cpu)
return;
if (finite_phase)
__sync_fetch_and_add(&nr_finite_running, 1);
else
__sync_fetch_and_add(&nr_inf_running, 1);
}
void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p)
{
if (bpf_get_smp_processor_id() == test_cpu && finite_phase)
__sync_fetch_and_add(&nr_finite_ticks, 1);
}
void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops nohz_tick_ops = {
.select_cpu = (void *)nohz_tick_select_cpu,
.enqueue = (void *)nohz_tick_enqueue,
.running = (void *)nohz_tick_running,
.tick = (void *)nohz_tick_tick,
.exit = (void *)nohz_tick_exit,
.name = "nohz_tick",
.timeout_ms = 1000U,
};

View File

@ -0,0 +1,347 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*
* Validate that a finite-slice EXT task restarts the scheduler tick when it
* follows an infinite-slice EXT task and an idle interval on a NOHZ_FULL CPU.
*/
#define _GNU_SOURCE
#include <bpf/bpf.h>
#include <errno.h>
#include <sched.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>
#include <scx/common.h>
#include "nohz_tick.bpf.skel.h"
#include "scx_test.h"
#ifndef SCHED_EXT
#define SCHED_EXT 7
#endif
#define MIN_FINITE_TICKS 3
#define PHASE_TIMEOUT_MS 1000
struct nohz_tick_ctx {
struct nohz_tick *skel;
cpu_set_t original_mask;
int test_cpu;
};
static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
{
int cpu;
for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++)
if (CPU_ISSET(cpu, mask))
return cpu;
return -1;
}
static int find_nohz_full_cpu(const cpu_set_t *allowed)
{
char buf[4096], *cur, *end;
FILE *file;
file = fopen("/sys/devices/system/cpu/nohz_full", "r");
if (!file)
return -1;
if (!fgets(buf, sizeof(buf), file)) {
fclose(file);
return -1;
}
fclose(file);
cur = buf;
while (*cur) {
long first, last;
int cpu;
while (*cur == ' ' || *cur == '\t' || *cur == ',')
cur++;
if (*cur < '0' || *cur > '9')
break;
errno = 0;
first = strtol(cur, &end, 10);
if (errno || end == cur || first < 0 || first >= CPU_SETSIZE)
return -1;
cur = end;
last = first;
if (*cur == '-') {
cur++;
errno = 0;
last = strtol(cur, &end, 10);
if (errno || end == cur || last < first)
return -1;
cur = end;
}
cpu = first_allowed_cpu(allowed, first, last);
if (cpu >= 0)
return cpu;
}
return -1;
}
static pid_t start_worker(int cpu)
{
struct sched_param param = {};
cpu_set_t mask;
pid_t parent;
pid_t pid;
parent = getpid();
pid = fork();
if (pid != 0)
return pid;
if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent)
_exit(1);
/*
* Become EXT before touching the target so it stays idle until wakeup.
*/
if (sched_setscheduler(0, SCHED_EXT, &param))
_exit(1);
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask))
_exit(1);
for (;;)
asm volatile("" ::: "memory");
}
static void stop_worker(pid_t pid)
{
if (pid <= 0)
return;
kill(pid, SIGKILL);
waitpid(pid, NULL, 0);
}
static int pause_worker(pid_t pid)
{
int status;
if (kill(pid, SIGSTOP))
return -errno;
if (waitpid(pid, &status, WUNTRACED) != pid)
return -errno;
if (!WIFSTOPPED(status))
return -ECHILD;
return 0;
}
static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms)
{
int elapsed;
for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value)
return true;
usleep(1000);
}
return false;
}
static enum scx_test_status setup(void **ctx_ptr)
{
struct nohz_tick_ctx *ctx;
cpu_set_t controller_mask;
int cpu;
ctx = calloc(1, sizeof(*ctx));
SCX_FAIL_IF(!ctx, "Failed to allocate context");
if (sched_getaffinity(0, sizeof(ctx->original_mask),
&ctx->original_mask)) {
free(ctx);
SCX_FAIL("Failed to get affinity (%d)", errno);
}
cpu = find_nohz_full_cpu(&ctx->original_mask);
if (cpu < 0) {
fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n");
free(ctx);
return SCX_TEST_SKIP;
}
controller_mask = ctx->original_mask;
CPU_CLR(cpu, &controller_mask);
if (CPU_COUNT(&controller_mask) == 0) {
fprintf(stderr, "SKIP: no housekeeping CPU available\n");
free(ctx);
return SCX_TEST_SKIP;
}
ctx->test_cpu = cpu;
ctx->skel = nohz_tick__open();
if (!ctx->skel) {
free(ctx);
SCX_FAIL("Failed to open skeleton");
}
SCX_ENUM_INIT(ctx->skel);
ctx->skel->rodata->test_cpu = cpu;
ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_ENQ_LAST;
if (nohz_tick__load(ctx->skel)) {
nohz_tick__destroy(ctx->skel);
free(ctx);
SCX_FAIL("Failed to load skeleton");
}
if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) {
nohz_tick__destroy(ctx->skel);
free(ctx);
SCX_FAIL("Failed to move controller off CPU %d (%d)", cpu, errno);
}
*ctx_ptr = ctx;
return SCX_TEST_PASS;
}
static enum scx_test_status run(void *ctx_ptr)
{
struct nohz_tick_ctx *ctx = ctx_ptr;
struct nohz_tick *skel = ctx->skel;
struct bpf_link *link = NULL;
enum scx_test_status status = SCX_TEST_FAIL;
pid_t finite_worker = -1;
pid_t inf_worker = -1;
u64 finite_running;
u64 finite_ticks;
int ret;
link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops);
if (!link) {
SCX_ERR("Failed to attach scheduler");
goto out;
}
/*
* Establish SCX_RQ_CAN_STOP_TICK with an infinite-slice task.
*/
inf_worker = start_worker(ctx->test_cpu);
if (inf_worker < 0) {
SCX_ERR("Failed to start infinite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_inf_running, 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Infinite-slice worker was not scheduled");
goto out;
}
/* Block without exiting so the rq retains the infinite-slice state. */
ret = pause_worker(inf_worker);
if (ret) {
SCX_ERR("Failed to stop infinite-slice worker (%d)", ret);
goto out;
}
/* Let the target enter idle with its tick stopped. */
usleep(100000);
/*
* The next EXT task receives a finite slice and must restart the tick.
*/
__atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE);
finite_worker = start_worker(ctx->test_cpu);
if (finite_worker < 0) {
SCX_ERR("Failed to start finite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_running, 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Finite-slice worker was not scheduled");
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Finite-slice worker received only %llu scheduler ticks",
(unsigned long long)skel->bss->nr_finite_ticks);
goto out;
}
stop_worker(finite_worker);
finite_worker = -1;
/*
* Leave the CPU idle after a finite-slice task. The next finite-slice
* task must restart the tick even though the slice type is unchanged.
*/
usleep(100000);
finite_running = __atomic_load_n(&skel->bss->nr_finite_running,
__ATOMIC_RELAXED);
finite_ticks = __atomic_load_n(&skel->bss->nr_finite_ticks,
__ATOMIC_RELAXED);
finite_worker = start_worker(ctx->test_cpu);
if (finite_worker < 0) {
SCX_ERR("Failed to start second finite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_running,
finite_running + 1, PHASE_TIMEOUT_MS)) {
SCX_ERR("Second finite-slice worker was not scheduled");
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_ticks,
finite_ticks + MIN_FINITE_TICKS,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Second finite-slice worker received only %llu scheduler ticks",
(unsigned long long)(skel->bss->nr_finite_ticks -
finite_ticks));
goto out;
}
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
(unsigned long long)skel->data->uei.kind,
(long long)skel->data->uei.exit_code);
goto out;
}
fprintf(stderr, "CPU %d received %llu finite-slice ticks\n",
ctx->test_cpu,
(unsigned long long)skel->bss->nr_finite_ticks);
status = SCX_TEST_PASS;
out:
stop_worker(finite_worker);
stop_worker(inf_worker);
if (link)
bpf_link__destroy(link);
return status;
}
static void cleanup(void *ctx_ptr)
{
struct nohz_tick_ctx *ctx = ctx_ptr;
sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask);
nohz_tick__destroy(ctx->skel);
free(ctx);
}
struct scx_test nohz_tick = {
.name = "nohz_tick",
.description = "Verify finite EXT slices restart the NOHZ_FULL tick",
.setup = setup,
.run = run,
.cleanup = cleanup,
};
REGISTER_SCX_TEST(&nohz_tick)