From 030db7005efe6a0705ddf07fced494c364a1c915 Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Tue, 7 Jul 2026 17:45:38 +0800 Subject: [PATCH 1/7] sched_ext: Documentation: Fix ops table header reference The "Where to Look" and "ABI Instability" sections state that the ops table is defined in include/linux/sched/ext.h. However, struct sched_ext_ops is actually defined in kernel/sched/ext/internal.h, along with the SCX_OPS_* flags; include/linux/sched/ext.h holds the core data structures (struct sched_ext_entity, struct scx_dispatch_q, ...) and the DSQ constants. Point the ops table references to the correct header. Signed-off-by: Liang Luo Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 4b1ffd03f516..2771ea4cc14a 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -493,8 +493,9 @@ a freshly woken up task gets on a CPU. Where to Look ============= -* ``include/linux/sched/ext.h`` defines the core data structures, ops table - and constants. +* ``include/linux/sched/ext.h`` defines the core data structures and + constants, while the ops table (``struct sched_ext_ops``) is defined in + ``kernel/sched/ext/internal.h``. * ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers. The functions prefixed with ``scx_bpf_`` can be called from the BPF @@ -555,7 +556,8 @@ ABI Instability =============== The APIs provided by sched_ext to BPF schedulers programs have no stability -guarantees. This includes the ops table callbacks and constants defined in +guarantees. This includes the ops table callbacks defined in +``kernel/sched/ext/internal.h`` and the constants defined in ``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in ``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``. From 18d62044cda7a2b40f59d910659c0b0d6accad37 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 8 Jul 2026 10:43:27 +0200 Subject: [PATCH 2/7] sched_ext: Preserve rq tracking across local DSQ dispatch dispatch_to_local_dsq() can run from scx_bpf_dsq_move_to_local() while ops.dispatch() has recorded the current rq. Moving a task to a local DSQ may switch to the source or destination rq before synchronously invoking ops.dequeue() through the following path: SCX_CALL_OP(dispatch, rq) ops.dispatch() scx_bpf_dsq_move_to_local() scx_flush_dispatch_buf() finish_dispatch() dispatch_to_local_dsq() scx_dispatch_enqueue() local_dsq_post_enq() call_task_dequeue() SCX_CALL_OP_TASK(dequeue, locked_rq, ...) The nested callback saves the recorded rq and restores it on return. If the rq tracking does not follow the lock switch, update_locked_rq() can trigger the following lockdep assertion while restoring an rq which is no longer held: WARNING: kernel/sched/sched.h:1641 at call_task_dequeue+0x160/0x170 Call Trace: scx_dispatch_enqueue+0x2b0/0x460 dispatch_to_local_dsq+0x138/0x230 scx_flush_dispatch_buf+0x1af/0x220 scx_bpf_dsq_move_to_local___v2+0xe2/0x1c0 bpf__sched_ext_ops_dispatch+0x4b/0xa7 do_pick_task_scx+0x3b6/0x910 __pick_next_task+0x105/0x1f0 __schedule+0x3e7/0x1980 Introduce switch_rq_lock() to update the tracking state together with each rq lock handoff. Use it in dispatch_to_local_dsq(), move_remote_task_to_local_dsq() and the in-balance paths of scx_dsq_move(), ensuring that scx_locked_rq() consistently refers to the rq whose lock is actually held throughout the lock dance. Fixes: 7fb39e4eb4c3 ("sched_ext: Save and restore scx_locked_rq across SCX_CALL_OP") Cc: stable@vger.kernel.org # 7.1+ Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e75e2fd5ab7e..ee66dacfc92c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -479,6 +479,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); +static void switch_rq_lock(struct rq *from, struct rq *to) +{ + bool tracked = scx_locked_rq() == from; + + if (tracked) + update_locked_rq(NULL); + raw_spin_rq_unlock(from); + raw_spin_rq_lock(to); + if (tracked) + update_locked_rq(to); +} + /* * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. @@ -2274,8 +2286,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, deactivate_task(src_rq, p, 0); set_task_cpu(p, cpu_of(dst_rq)); - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(dst_rq); + switch_rq_lock(src_rq, dst_rq); /* * We want to pass scx-specific enq_flags but activate_task() will @@ -2608,9 +2619,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, /* switch to @src_rq lock */ if (locked_rq != src_rq) { - raw_spin_rq_unlock(locked_rq); + switch_rq_lock(locked_rq, src_rq); locked_rq = src_rq; - raw_spin_rq_lock(src_rq); } /* task_rq couldn't have changed if we're still the holding cpu */ @@ -2644,10 +2654,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* switch back to @rq lock */ - if (locked_rq != rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(rq); - } + if (locked_rq != rq) + switch_rq_lock(locked_rq, rq); } /** @@ -8825,10 +8833,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; if (in_balance) { - if (this_rq != src_rq) { - raw_spin_rq_unlock(this_rq); - raw_spin_rq_lock(src_rq); - } + if (this_rq != src_rq) + switch_rq_lock(this_rq, src_rq); } else { raw_spin_rq_lock(src_rq); } @@ -8860,10 +8866,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, dispatched = true; out: if (in_balance) { - if (this_rq != locked_rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(this_rq); - } + if (this_rq != locked_rq) + switch_rq_lock(locked_rq, this_rq); } else { raw_spin_rq_unlock_irqrestore(locked_rq, flags); } From 4ec10f38ff901dc10503d57cbdcf941248419ac1 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 8 Jul 2026 09:46:48 +0200 Subject: [PATCH 3/7] sched_ext: Enable tick for finite slices on nohz_full set_next_task_scx() updates the tick dependency before __schedule() updates rq->curr. When switching from a non-EXT task, such as idle, to an EXT task with a finite slice, sched_update_tick_dependency() checks the outgoing task and can allow the tick to remain stopped. The dependency can also be lost without a slice-type transition. After a finite-slice task leaves the CPU idle, the enqueue path can clear the dependency against the idle rq->curr. SCX_RQ_CAN_STOP_TICK still records a finite slice, so another finite task skips the transition block and can run without the ticks needed to expire its slice. The reverse mismatch can also happen when the last finite-slice EXT task is dequeued: sub_nr_running() updates the dependency before rq->curr changes, so the outgoing task state can keep the dependency set after the CPU goes idle. Fix this by unconditionally enabling the scheduler tick whenever a finite-slice EXT task is selected on a nohz_full CPU. Moreover, when the last runnable EXT task leaves, ignore the outgoing EXT slice state so the generic scheduler can correctly re-evaluate and clear the tick dependency. Fixes: 22a920209ab6 ("sched_ext: Implement tickless support") Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 47 +++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ee66dacfc92c..2dc6977d7984 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2986,24 +2986,38 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) /* * @p is getting newly scheduled or got kicked after someone updated its - * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). + * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be + * stopped. See scx_can_stop_tick(). + * + * Moreover, refresh the load_avgs just when transitioning in and out of + * nohz. In the future, we might want to add a mechanism to update + * load_avgs periodically on tick-stopped CPUs. */ - if ((p->scx.slice == SCX_SLICE_INF) != - (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { - if (p->scx.slice == SCX_SLICE_INF) + if (p->scx.slice == SCX_SLICE_INF) { + if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { + /* + * Bypass mode always assigns finite slices, so @p + * can't have an infinite slice while bypassing. + * Therefore, sched_update_tick_dependency() can safely + * evaluate the outgoing task. + */ rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; - else - rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + sched_update_tick_dependency(rq); - sched_update_tick_dependency(rq); + update_other_load_avgs(rq); + } + } else { + if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + update_other_load_avgs(rq); + } /* - * For now, let's refresh the load_avgs just when transitioning - * in and out of nohz. In the future, we might want to add a - * mechanism which calls the following periodically on - * tick-stopped CPUs. + * @rq still references the outgoing scheduling context. A finite + * slice is sufficient by itself to require the tick. */ - update_other_load_avgs(rq); + if (tick_nohz_full_cpu(cpu_of(rq))) + tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); } } @@ -4329,6 +4343,15 @@ bool scx_can_stop_tick(struct rq *rq) if (p->sched_class != &ext_sched_class) return true; + /* + * @rq->curr may still reference an outgoing EXT task after it has been + * dequeued. If no EXT tasks are accounted on @rq, ignore its stale + * slice state. If another task is dispatched from a DSQ, + * set_next_task_scx() will update the dependency for the incoming task. + */ + if (!rq->scx.nr_running) + return true; + if (scx_bypassing(sch, cpu_of(rq))) return false; From cfe950d79f524e72bc263f2b153a6d905a75a794 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 8 Jul 2026 09:46:49 +0200 Subject: [PATCH 4/7] selftests/sched_ext: Verify nohz_full tick behavior Finite-slice EXT tasks need the periodic scheduler tick to expire their slices even when nohz_full is enabled. Add a regression test that selects a nohz_full CPU and exercises both infinite-to-finite and finite-to-finite slice transitions across an idle interval. For each finite task, verify that its ops.tick() callback is invoked. Skip the test when an allowed nohz_full CPU and a separate housekeeping CPU are not available. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + .../selftests/sched_ext/nohz_tick.bpf.c | 65 ++++ tools/testing/selftests/sched_ext/nohz_tick.c | 347 ++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/nohz_tick.bpf.c create mode 100644 tools/testing/selftests/sched_ext/nohz_tick.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 5d2dffca0e91..3cfe90e0f34f 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -176,6 +176,7 @@ auto-test-targets := \ maybe_null \ minimal \ non_scx_kfunc_deny \ + nohz_tick \ numa \ allowed_cpus \ peek_dsq \ diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c new file mode 100644 index 000000000000..6998c5dd6bcb --- /dev/null +++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Exercise tick dependency transitions between infinite and finite slices. + */ +#include + +char _license[] SEC("license") = "GPL"; + +const volatile s32 test_cpu; +bool finite_phase; +u64 nr_inf_running; +u64 nr_finite_running; +u64 nr_finite_ticks; + +UEI_DEFINE(uei); + +s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu, + u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags) +{ + u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF; + + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags); + if (enq_flags & SCX_ENQ_LAST) + scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p) +{ + if (bpf_get_smp_processor_id() != test_cpu) + return; + + if (finite_phase) + __sync_fetch_and_add(&nr_finite_running, 1); + else + __sync_fetch_and_add(&nr_inf_running, 1); +} + +void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p) +{ + if (bpf_get_smp_processor_id() == test_cpu && finite_phase) + __sync_fetch_and_add(&nr_finite_ticks, 1); +} + +void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops nohz_tick_ops = { + .select_cpu = (void *)nohz_tick_select_cpu, + .enqueue = (void *)nohz_tick_enqueue, + .running = (void *)nohz_tick_running, + .tick = (void *)nohz_tick_tick, + .exit = (void *)nohz_tick_exit, + .name = "nohz_tick", + .timeout_ms = 1000U, +}; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c new file mode 100644 index 000000000000..028f54391c2c --- /dev/null +++ b/tools/testing/selftests/sched_ext/nohz_tick.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Validate that a finite-slice EXT task restarts the scheduler tick when it + * follows an infinite-slice EXT task and an idle interval on a NOHZ_FULL CPU. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "nohz_tick.bpf.skel.h" +#include "scx_test.h" + +#ifndef SCHED_EXT +#define SCHED_EXT 7 +#endif + +#define MIN_FINITE_TICKS 3 +#define PHASE_TIMEOUT_MS 1000 + +struct nohz_tick_ctx { + struct nohz_tick *skel; + cpu_set_t original_mask; + int test_cpu; +}; + +static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) +{ + int cpu; + + for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, mask)) + return cpu; + + return -1; +} + +static int find_nohz_full_cpu(const cpu_set_t *allowed) +{ + char buf[4096], *cur, *end; + FILE *file; + + file = fopen("/sys/devices/system/cpu/nohz_full", "r"); + if (!file) + return -1; + if (!fgets(buf, sizeof(buf), file)) { + fclose(file); + return -1; + } + fclose(file); + + cur = buf; + while (*cur) { + long first, last; + int cpu; + + while (*cur == ' ' || *cur == '\t' || *cur == ',') + cur++; + if (*cur < '0' || *cur > '9') + break; + + errno = 0; + first = strtol(cur, &end, 10); + if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) + return -1; + cur = end; + last = first; + if (*cur == '-') { + cur++; + errno = 0; + last = strtol(cur, &end, 10); + if (errno || end == cur || last < first) + return -1; + cur = end; + } + + cpu = first_allowed_cpu(allowed, first, last); + if (cpu >= 0) + return cpu; + } + + return -1; +} + +static pid_t start_worker(int cpu) +{ + struct sched_param param = {}; + cpu_set_t mask; + pid_t parent; + pid_t pid; + + parent = getpid(); + pid = fork(); + if (pid != 0) + return pid; + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) + _exit(1); + + /* + * Become EXT before touching the target so it stays idle until wakeup. + */ + if (sched_setscheduler(0, SCHED_EXT, ¶m)) + _exit(1); + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask)) + _exit(1); + + for (;;) + asm volatile("" ::: "memory"); +} + +static void stop_worker(pid_t pid) +{ + if (pid <= 0) + return; + + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); +} + +static int pause_worker(pid_t pid) +{ + int status; + + if (kill(pid, SIGSTOP)) + return -errno; + if (waitpid(pid, &status, WUNTRACED) != pid) + return -errno; + if (!WIFSTOPPED(status)) + return -ECHILD; + + return 0; +} + +static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) +{ + int elapsed; + + for (elapsed = 0; elapsed < timeout_ms; elapsed++) { + if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value) + return true; + usleep(1000); + } + + return false; +} + +static enum scx_test_status setup(void **ctx_ptr) +{ + struct nohz_tick_ctx *ctx; + cpu_set_t controller_mask; + int cpu; + + ctx = calloc(1, sizeof(*ctx)); + SCX_FAIL_IF(!ctx, "Failed to allocate context"); + if (sched_getaffinity(0, sizeof(ctx->original_mask), + &ctx->original_mask)) { + free(ctx); + SCX_FAIL("Failed to get affinity (%d)", errno); + } + + cpu = find_nohz_full_cpu(&ctx->original_mask); + if (cpu < 0) { + fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n"); + free(ctx); + return SCX_TEST_SKIP; + } + + controller_mask = ctx->original_mask; + CPU_CLR(cpu, &controller_mask); + if (CPU_COUNT(&controller_mask) == 0) { + fprintf(stderr, "SKIP: no housekeeping CPU available\n"); + free(ctx); + return SCX_TEST_SKIP; + } + + ctx->test_cpu = cpu; + ctx->skel = nohz_tick__open(); + if (!ctx->skel) { + free(ctx); + SCX_FAIL("Failed to open skeleton"); + } + + SCX_ENUM_INIT(ctx->skel); + ctx->skel->rodata->test_cpu = cpu; + ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL | + SCX_OPS_ENQ_LAST; + if (nohz_tick__load(ctx->skel)) { + nohz_tick__destroy(ctx->skel); + free(ctx); + SCX_FAIL("Failed to load skeleton"); + } + + if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) { + nohz_tick__destroy(ctx->skel); + free(ctx); + SCX_FAIL("Failed to move controller off CPU %d (%d)", cpu, errno); + } + + *ctx_ptr = ctx; + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx_ptr) +{ + struct nohz_tick_ctx *ctx = ctx_ptr; + struct nohz_tick *skel = ctx->skel; + struct bpf_link *link = NULL; + enum scx_test_status status = SCX_TEST_FAIL; + pid_t finite_worker = -1; + pid_t inf_worker = -1; + u64 finite_running; + u64 finite_ticks; + int ret; + + link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + + /* + * Establish SCX_RQ_CAN_STOP_TICK with an infinite-slice task. + */ + inf_worker = start_worker(ctx->test_cpu); + if (inf_worker < 0) { + SCX_ERR("Failed to start infinite-slice worker (%d)", errno); + goto out; + } + if (!wait_for_counter(&skel->bss->nr_inf_running, 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Infinite-slice worker was not scheduled"); + goto out; + } + + /* Block without exiting so the rq retains the infinite-slice state. */ + ret = pause_worker(inf_worker); + if (ret) { + SCX_ERR("Failed to stop infinite-slice worker (%d)", ret); + goto out; + } + + /* Let the target enter idle with its tick stopped. */ + usleep(100000); + + /* + * The next EXT task receives a finite slice and must restart the tick. + */ + __atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE); + finite_worker = start_worker(ctx->test_cpu); + if (finite_worker < 0) { + SCX_ERR("Failed to start finite-slice worker (%d)", errno); + goto out; + } + if (!wait_for_counter(&skel->bss->nr_finite_running, 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Finite-slice worker was not scheduled"); + goto out; + } + if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Finite-slice worker received only %llu scheduler ticks", + (unsigned long long)skel->bss->nr_finite_ticks); + goto out; + } + stop_worker(finite_worker); + finite_worker = -1; + + /* + * Leave the CPU idle after a finite-slice task. The next finite-slice + * task must restart the tick even though the slice type is unchanged. + */ + usleep(100000); + finite_running = __atomic_load_n(&skel->bss->nr_finite_running, + __ATOMIC_RELAXED); + finite_ticks = __atomic_load_n(&skel->bss->nr_finite_ticks, + __ATOMIC_RELAXED); + + finite_worker = start_worker(ctx->test_cpu); + if (finite_worker < 0) { + SCX_ERR("Failed to start second finite-slice worker (%d)", errno); + goto out; + } + if (!wait_for_counter(&skel->bss->nr_finite_running, + finite_running + 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Second finite-slice worker was not scheduled"); + goto out; + } + if (!wait_for_counter(&skel->bss->nr_finite_ticks, + finite_ticks + MIN_FINITE_TICKS, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Second finite-slice worker received only %llu scheduler ticks", + (unsigned long long)(skel->bss->nr_finite_ticks - + finite_ticks)); + goto out; + } + + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", + (unsigned long long)skel->data->uei.kind, + (long long)skel->data->uei.exit_code); + goto out; + } + + fprintf(stderr, "CPU %d received %llu finite-slice ticks\n", + ctx->test_cpu, + (unsigned long long)skel->bss->nr_finite_ticks); + status = SCX_TEST_PASS; +out: + stop_worker(finite_worker); + stop_worker(inf_worker); + if (link) + bpf_link__destroy(link); + return status; +} + +static void cleanup(void *ctx_ptr) +{ + struct nohz_tick_ctx *ctx = ctx_ptr; + + sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask); + nohz_tick__destroy(ctx->skel); + free(ctx); +} + +struct scx_test nohz_tick = { + .name = "nohz_tick", + .description = "Verify finite EXT slices restart the NOHZ_FULL tick", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; +REGISTER_SCX_TEST(&nohz_tick) From db4e9defd2e8620abee04cfe5809c0bcd6ecf06a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 11:08:13 -1000 Subject: [PATCH 5/7] sched_ext: Record an error on errno-only sub-enable failure scx_sub_enable_workfn() has several failure paths that only return an errno (e.g. -ENOMEM from an allocation) and jump to err_disable without calling scx_error(). scx_flush_disable_work() runs the disable, and thus ops.exit(), only when an error has been recorded, so an errno-only failure leaves the half-initialized sub-scheduler linked. Record an error at the err_disable sink so every errno-only failure runs the disable path. Fixes: ebeca1f930ea ("sched_ext: Introduce cgroup sub-sched support") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2dc6977d7984..40cef77394ac 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7746,6 +7746,12 @@ static void scx_sub_enable_workfn(struct kthread_work *work) percpu_up_write(&scx_fork_rwsem); err_disable: mutex_unlock(&scx_enable_mutex); + /* + * Some enable failures only return an errno (e.g. -ENOMEM from an + * allocation) without calling scx_error(). Record it so + * scx_flush_disable_work() runs the disable and ops.exit() fires. + */ + scx_error(sch, "scx_sub_enable() failed (%d)", ret); scx_flush_disable_work(sch); cmd->ret = 0; } From 49b3378a750cf85112e656d003145d4b5d0da232 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 11:08:22 -1000 Subject: [PATCH 6/7] sched_ext: Fix premature ops->priv publication in scx_alloc_and_add_sched() scx_alloc_and_add_sched() publishes @sch through ops->priv before allocating the cgroup path. If that allocation fails, the unwind path clears ops->priv and frees @sch immediately. scx_prog_sched() callers can dereference ops->priv from RCU context the moment it is set, so freeing without a grace period can use-after-free a concurrent kfunc caller. Move the publication below the cgroup path allocation so that every failure path after publication frees @sch through kobject_put(), whose release path defers the freeing by a grace period. Fixes: 105dcd005be2 ("sched_ext: Introduce scx_prog_sched()") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 40cef77394ac..aeee44e016ab 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6878,11 +6878,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, sch->ops = *cmd->ops; } - rcu_assign_pointer(ops->priv, sch); - - sch->kobj.kset = scx_kset; - INIT_LIST_HEAD(&sch->all); - #ifdef CONFIG_EXT_SUB_SCHED char *buf = kzalloc(PATH_MAX, GFP_KERNEL); if (!buf) { @@ -6900,7 +6895,19 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, sch->cgrp = cgrp; INIT_LIST_HEAD(&sch->children); INIT_LIST_HEAD(&sch->sibling); +#endif /* CONFIG_EXT_SUB_SCHED */ + /* + * Publishing makes @sch visible to scx_prog_sched() readers. Failure + * paths after this point must free @sch through kobject_put() whose + * release path defers the actual freeing by an RCU grace period. + */ + rcu_assign_pointer(ops->priv, sch); + + sch->kobj.kset = scx_kset; + INIT_LIST_HEAD(&sch->all); + +#ifdef CONFIG_EXT_SUB_SCHED if (parent) { /* * Pin @parent for @sch's lifetime. The kobject hierarchy pins @@ -6955,7 +6962,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, #ifdef CONFIG_EXT_SUB_SCHED err_free_lb_resched: - RCU_INIT_POINTER(ops->priv, NULL); free_cpumask_var(sch->bypass_lb_resched_cpumask); #endif err_free_lb_cpumask: From e6979d05c6a6fe79980f08d63f039f0b27c30a1c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 11:08:41 -1000 Subject: [PATCH 7/7] tools/sched_ext: scx - Fix cmask_subset(), cmask_equal() and cmask_weight() cmask_equal(), cmask_weight() and cmask_subset() bounded their word walks with CMASK_NR_WORDS(nr_cids), which pads by one word and can't tell the last word in use without @base. The walks could thus cover a slack word past the active range, which cmask_reframe() leaves non-zero: a stale bit there gave cmask_equal() a spurious mismatch, cmask_weight() an inflated count, and cmask_subset() a spurious violation. cmask_subset() could also read @b->bits[] one word past its allocation (within the arena's fault-recovered range, so harmless), and deviated from the kernel scx_cmask_subset() by failing any @a range that doesn't nest inside @b's even when the overhanging bits are all clear. Bound the cmask_equal() and cmask_weight() walks by the words the range actually spans, with early returns for empty ranges. Rewrite cmask_subset() to match the kernel semantics: scan @a's overhangs for set bits with cmask_next_set() and walk the words of the range intersection. cmask_subset() moves below cmask_next_set(), which it now uses. Padding bits don't need masking as every cmask helper keeps them clear. Fixes: a58e6b79b432 ("sched_ext: Add cmask, a base-windowed bitmap over cid space") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/include/scx/cid.bpf.h | 88 +++++++++++++++++---------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index db247e42fb45..6b0b4e41b288 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a, if (a->base != b->base || a->nr_cids != b->nr_cids) return false; - nr_words = CMASK_NR_WORDS(a->nr_cids); + if (a->nr_cids == 0) + return true; + nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1; bpf_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) @@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a, return true; } -/* - * True iff every bit set in @a is also set in @b over the intersection of - * their ranges. Bits of @a outside @b's range fail the test. - */ -static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, - const struct scx_cmask __arena *b) -{ - u32 a_end = a->base + a->nr_cids; - u32 b_end = b->base + b->nr_cids; - u32 a_wbase = a->base / 64; - u32 b_wbase = b->base / 64; - u32 nr_words, i; - - /* any bit of @a outside @b's range is a subset violation */ - if (a->base < b->base || a_end > b_end) - return false; - - nr_words = CMASK_NR_WORDS(a->nr_cids); - bpf_for(i, 0, CMASK_MAX_WORDS) { - u32 wi_b; - - if (i >= nr_words) - break; - wi_b = a_wbase + i - b_wbase; - if (a->bits[i] & ~b->bits[wi_b]) - return false; - } - return true; -} - /** * cmask_next_set - find the first set bit at or after @cid * @m: cmask to search @@ -488,16 +460,66 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m) (cid) < (m)->base + (m)->nr_cids; \ (cid) = cmask_next_set((m), (cid) + 1)) +/* + * True iff every bit set in @a is also set in @b. Matches the kernel-side + * scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside + * @b's range fail the test. + */ +static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b) +{ + u32 a_end = a->base + a->nr_cids; + u32 b_end = b->base + b->nr_cids; + u32 a_wbase = a->base / 64; + u32 b_wbase = b->base / 64; + u32 lo = a->base > b->base ? a->base : b->base; + u32 hi = a_end < b_end ? a_end : b_end; + u32 lo_word, hi_word, i; + + /* set bits of @a outside @b's range can't be in @b */ + if (a->base < b->base && + cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end)) + return false; + if (a_end > b_end && + cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end) + return false; + + if (lo >= hi) + return true; + + /* + * Walk the words the range intersection spans. Plain word tests + * suffice: the scans above guarantee @a has no set bit outside @b's + * range and padding bits are kept clear by all cmask helpers. + */ + lo_word = lo / 64; + hi_word = (hi - 1) / 64; + + bpf_for(i, 0, CMASK_MAX_WORDS) { + u32 w = lo_word + i; + + if (w > hi_word) + break; + if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase]) + return false; + } + return true; +} + /* * Population count over [base, base + nr_cids). Padding bits in the head/tail * words are guaranteed zero by the mutating helpers, so a flat popcount over - * all words is correct. + * the words the range spans is correct. */ static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) { - u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i; + u32 nr_words, i; u32 count = 0; + if (!m->nr_cids) + return 0; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + bpf_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break;