From 5f01293930d18f8473681b378bb483b7087fc0dd Mon Sep 17 00:00:00 2001 From: Changwoo Min Date: Wed, 19 Aug 2026 01:04:29 +0900 Subject: [PATCH 01/12] sched_ext: Allow ops.cgroup_set_bandwidth() to be sleepable ops.cgroup_set_bandwidth() is delivered from scx_group_set_bandwidth(), which runs from the cpu.max cgroup interface write path (tg_set_bandwidth()) in process context. scx_group_set_bandwidth() holds percpu_down_read(&scx_cgroup_ops_rwsem), whose read side may sleep. The call site is therefore sleepable, like ops.cgroup_init(). bpf_scx_check_member() rejects a sleepable program on any member not on its allow-list, so a BPF scheduler cannot allocate -- which is sleepable -- when a cgroup gains a cpu.max limit at runtime; it must instead pre-reserve memory for a callback that cannot allocate. Add cgroup_set_bandwidth() to the allow-list so the callback can allocate on demand, and document that it may block. A scheduler must decide at load time whether to mark the callback sleepable, but the allow-list entry is a verifier property with no symbol to probe. Add a compatibility marker whose presence in the kernel's BTF lets userspace detect this support: DEFINE_SCX_COMPAT_MARKER() emits an empty, callerless function, here scx_compat_marker_cgroup_set_bandwidth_may_sleep(). It is __used __retain so neither the compiler nor the linker (under CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) drops it. The markers share the scx_compat_marker_ prefix and are collected near the end of ext.c so more can be added as further capabilities appear. Signed-off-by: Changwoo Min Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 14 ++++++++++++++ kernel/sched/ext/internal.h | 23 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 10af28a9f2c0..b646711a45fe 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8079,6 +8079,7 @@ static int bpf_scx_check_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, cgroup_init): case offsetof(struct sched_ext_ops, cgroup_exit): case offsetof(struct sched_ext_ops, cgroup_prep_move): + case offsetof(struct sched_ext_ops, cgroup_set_bandwidth): #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): @@ -11041,3 +11042,16 @@ static int __init scx_init(void) return 0; } __initcall(scx_init); + +/* + * Compatibility markers for userspace. Existence of a marker function + * represents that the kernel supports that sched-ext feature. + */ + +/* + * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that + * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback. + */ +#ifdef CONFIG_EXT_GROUP_SCHED +DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep); +#endif /* CONFIG_EXT_GROUP_SCHED */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 27bbf5e04d90..53e136a47924 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -753,7 +753,7 @@ struct sched_ext_ops { * @burst_us: bandwidth control burst * * Update @cgrp's bandwidth control parameters. This is from the cpu.max - * cgroup interface. + * cgroup interface. This operation may block. * * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled * to. For example, if @period_us is 1_000_000 and @quota_us is @@ -2001,6 +2001,27 @@ struct scx_bstr_buf { char line[SCX_EXIT_MSG_LEN]; }; +/* Internal helper for DEFINE_SCX_COMPAT_MARKER(). */ +#define DECLARE_SCX_COMPAT_MARKER(func) \ + extern void scx_compat_marker_##func(void) + +/** + * DEFINE_SCX_COMPAT_MARKER() - define a userspace capability marker + * @func: marker suffix; the defined symbol is scx_compat_marker_@func + * + * Emit an empty, callerless function that is retained in the kernel's BTF. + * Its presence is part of the kernel<->userspace contract: userspace probes + * scx_compat_marker_@func (e.g. via BTF) to detect that this kernel supports + * the corresponding feature. + * + * The leading declaration suppresses the missing-prototype warning; the + * trailing declaration consumes the semicolon at the use site. + */ +#define DEFINE_SCX_COMPAT_MARKER(func) \ + DECLARE_SCX_COMPAT_MARKER(func); \ + __used __retain void scx_compat_marker_##func(void) {} \ + DECLARE_SCX_COMPAT_MARKER(func) + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); From 4fb8d6379d2c7ceecb2b3e111954d29089d59492 Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Wed, 19 Aug 2026 11:12:44 +0800 Subject: [PATCH 02/12] sched_ext: Fix nonexistent field in sched-ext.rst example The ops.exit() example in sched-ext.rst reads ei->type, but struct scx_exit_info has never had a type field - the exit reason is exposed as ei->kind since the struct was introduced. A scheduler written following the example fails to compile with error: no member named 'type' in 'struct scx_exit_info' Use ei->kind. Fixes: fa48e8d2c7b5 ("sched_ext: Documentation: scheduler: Document extensible scheduler class") Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 0e97fd019994..35b550671ca7 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -230,7 +230,7 @@ optional. The following modified excerpt is from void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) { - exit_type = ei->type; + exit_type = ei->kind; } SEC(".struct_ops") From 72c5ae18ebe6588101f2c1e96be61618ce06f182 Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Thu, 20 Aug 2026 10:37:44 +0800 Subject: [PATCH 03/12] Docs/admin-guide/cgroup-v2: document BPF scheduler callbacks for cpu.max and cpu.idle The cpu.weight and cpu.weight.nice entries already state that the files also affect a BPF scheduler through the cgroup_set_weight callback. However, cpu.max, cpu.max.burst and cpu.idle only mention the fair-class scheduler, even though sched_ext implements the cgroup_set_bandwidth (notified with the period/quota from cpu.max and the burst from cpu.max.burst) and cgroup_set_idle callbacks from these interfaces. Mirror the cpu.weight wording for the three entries and generalize the category preamble to refer to the corresponding cgroup_set_* callback so it keeps covering the entries below. Suggested-by: Tejun Heo Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index aed195a71cbf..3dc6889ebdb2 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1130,9 +1130,9 @@ policy and the underlying scheduler. From the point of view of the cpu controlle processes can be categorized as follows: * Processes under the fair-class scheduler -* Processes under a BPF scheduler with the ``cgroup_set_weight`` callback +* Processes under a BPF scheduler with the corresponding ``cgroup_set_*`` callback * Everything else: ``SCHED_{FIFO,RR,DEADLINE}`` and processes under a BPF scheduler - without the ``cgroup_set_weight`` callback + without the corresponding ``cgroup_set_*`` callback For details on when a process is under the fair-class scheduler or a BPF scheduler, check out :ref:`Documentation/scheduler/sched-ext.rst `. @@ -1202,7 +1202,9 @@ will be referred to. All time durations are in microseconds. $PERIOD duration. "max" for $MAX indicates no limit. If only one number is written, $MAX is updated. - This file affects only processes under the fair-class scheduler. + This file affects only processes under the fair-class scheduler and a BPF + scheduler with the ``cgroup_set_bandwidth`` callback depending on what + the callback actually does. cpu.max.burst A read-write single value file which exists on non-root @@ -1210,7 +1212,9 @@ will be referred to. All time durations are in microseconds. The burst in the range [0, $MAX]. - This file affects only processes under the fair-class scheduler. + This file affects only processes under the fair-class scheduler and a BPF + scheduler with the ``cgroup_set_bandwidth`` callback depending on what + the callback actually does. cpu.pressure A read-write nested-keyed file. @@ -1262,7 +1266,9 @@ will be referred to. All time durations are in microseconds. own relative priorities, but the cgroup itself will be treated as very low priority relative to its peers. - This file affects only processes under the fair-class scheduler. + This file affects only processes under the fair-class scheduler and a BPF + scheduler with the ``cgroup_set_idle`` callback depending on what the + callback actually does. Memory ------ From e10b8b4931e10dbcce5b369583461d81c69187e8 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 18 Aug 2026 09:48:40 -1000 Subject: [PATCH 04/12] sched_ext: Sync tools autogen enum headers from the scx repo Regenerate enum_defs.autogen.h against the current tree, picking up the dispatch verdict enums and dropping the marker for the removed SCX_RQ_IN_BALANCE. Add enums_abi.autogen.h, a table of 64-bit scx enumerator values generated from vmlinux.h, used as the substitution source when the running kernel's BTF truncates 64-bit enum values to 32 bits. Signed-off-by: Tejun Heo --- .../sched_ext/include/scx/enum_defs.autogen.h | 5 +- .../sched_ext/include/scx/enums_abi.autogen.h | 223 ++++++++++++++++++ 2 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 tools/sched_ext/include/scx/enums_abi.autogen.h diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 19aa1de3e700..63b6b14b19bd 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -56,6 +56,10 @@ #define HAVE_SCX_DEQ_SLEEP #define HAVE_SCX_DEQ_CORE_SCHED_EXEC #define HAVE_SCX_DEQ_SCHED_CHANGE +#define HAVE_SCX_DSP_NONE +#define HAVE_SCX_DSP_LOCAL +#define HAVE_SCX_DSP_PREV +#define HAVE_SCX_DSP_RETRY #define HAVE_SCX_DSQ_FLAG_BUILTIN #define HAVE_SCX_DSQ_FLAG_LOCAL_ON #define HAVE_SCX_DSQ_INVALID @@ -188,7 +192,6 @@ #define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY #define HAVE_SCX_RQ_IN_WAKEUP -#define HAVE_SCX_RQ_IN_BALANCE #define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING #define HAVE_SCX_SLICE_OOB_DUR_BITS diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h new file mode 100644 index 000000000000..d53899764f5a --- /dev/null +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -0,0 +1,223 @@ +/* + * WARNING: This file is autogenerated from gen_enum_defs.py [1]. + * + * scx enumerator values from the vmlinux.h this tree is built against. + * Used as the substitution source when the running kernel's BTF lacks + * BTF_KIND_ENUM64 encoding and 64-bit enum values are truncated. + * + * [1] https://github.com/sched-ext/scx/blob/main/scripts/gen_enum_defs.py + */ + +#ifndef __ENUMS_ABI_AUTOGEN_H__ +#define __ENUMS_ABI_AUTOGEN_H__ + +struct __scx_enum_abi_val { + const char *type; + const char *name; + u64 val; +}; + +static const struct __scx_enum_abi_val __scx_enum_abi_vals[] + __attribute__((unused)) = { + { "scx_arena_consts", "SCX_ARENA_MIN_ORDER", 0x3LLU }, + { "scx_arena_consts", "SCX_ARENA_GROW_PAGES", 0x4LLU }, + { "scx_cap_flags", "__SCX_CAP_ENQ_IMMED", 0x0LLU }, + { "scx_cap_flags", "__SCX_CAP_ENQ", 0x1LLU }, + { "scx_cap_flags", "__SCX_CAP_PREEMPT", 0x2LLU }, + { "scx_cap_flags", "__SCX_CAP_PERF", 0x3LLU }, + { "scx_cap_flags", "__SCX_NR_CAPS", 0x4LLU }, + { "scx_cap_flags", "__SCX_CAP_ALL", 0xfLLU }, + { "scx_cap_flags", "SCX_CAP_ENQ_IMMED", 0x1LLU }, + { "scx_cap_flags", "SCX_CAP_ENQ", 0x2LLU }, + { "scx_cap_flags", "SCX_CAP_PREEMPT", 0x4LLU }, + { "scx_cap_flags", "SCX_CAP_PERF", 0x8LLU }, + { "scx_cap_flags", "SCX_CAP_BASE", 0x1LLU }, + { "scx_cap_flags", "SCX_CAPS_REENQ_ON_LOSS", 0x3LLU }, + { "scx_cid_consts", "SCX_CID_SHARD_SIZE_DFL", 0x18LLU }, + { "scx_cid_consts", "SCX_CID_SHARD_MAX_CPUS", 0x200LLU }, + { "scx_consts", "SCX_DSP_DFL_MAX_BATCH", 0x20LLU }, + { "scx_consts", "SCX_DSP_MAX_LOOPS", 0x20LLU }, + { "scx_consts", "SCX_WATCHDOG_MAX_TIMEOUT", 0x7530LLU }, + { "scx_consts", "SCX_RESCUE_DFL_BW_PPT", 0x14LLU }, + { "scx_consts", "SCX_RESCUE_MAX_BW_PPT", 0xfaLLU }, + { "scx_consts", "SCX_RESCUE_DISABLE", 0xffffffffLLU }, + { "scx_consts", "SCX_RESCUE_DFL_QUANTUM_US", 0x1388LLU }, + { "scx_consts", "SCX_RESCUE_MIN_QUANTUM_US", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_MAX_QUANTUM_US", 0x186a0LLU }, + { "scx_consts", "SCX_RESCUE_MIN_SLICE_US", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_OVERLOAD_MULT", 0x10LLU }, + { "scx_consts", "SCX_RESCUE_MIN_OVERLOAD_MS", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_MAX_OVERLOAD_MS", 0x3a98LLU }, + { "scx_consts", "SCX_TID_CHUNK", 0x400LLU }, + { "scx_consts", "SCX_EXIT_BT_LEN", 0x40LLU }, + { "scx_consts", "SCX_EXIT_MSG_LEN", 0x400LLU }, + { "scx_consts", "SCX_EXIT_DUMP_DFL_LEN", 0x8000LLU }, + { "scx_consts", "SCX_CPUPERF_ONE", 0x400LLU }, + { "scx_consts", "SCX_TASK_ITER_BATCH", 0x20LLU }, + { "scx_consts", "SCX_BYPASS_HOST_NTH", 0x2LLU }, + { "scx_consts", "SCX_BYPASS_LB_DFL_INTV_US", 0x7a120LLU }, + { "scx_consts", "SCX_BYPASS_LB_DONOR_PCT", 0x7dLLU }, + { "scx_consts", "SCX_BYPASS_LB_MIN_DELTA_DIV", 0x4LLU }, + { "scx_consts", "SCX_BYPASS_LB_BATCH", 0x100LLU }, + { "scx_consts", "SCX_REENQ_MAX_REPEAT", 0x100LLU }, + { "scx_consts", "SCX_SUB_MAX_DEPTH", 0x4LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_RT", 0x0LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_DL", 0x1LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_STOP", 0x2LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_UNKNOWN", 0x3LLU }, + { "scx_deq_flags", "SCX_DEQ_SLEEP", 0x1LLU }, + { "scx_deq_flags", "SCX_DEQ_CORE_SCHED_EXEC", 0x100000000LLU }, + { "scx_deq_flags", "SCX_DEQ_SCHED_CHANGE", 0x200000000LLU }, + { "scx_dsp_verdict", "SCX_DSP_NONE", 0x0LLU }, + { "scx_dsp_verdict", "SCX_DSP_LOCAL", 0x1LLU }, + { "scx_dsp_verdict", "SCX_DSP_PREV", 0x2LLU }, + { "scx_dsp_verdict", "SCX_DSP_RETRY", 0x3LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_FLAG_BUILTIN", 0x8000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_FLAG_LOCAL_ON", 0x4000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_INVALID", 0x8000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_GLOBAL", 0x8000000000000001LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL", 0x8000000000000002LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_BYPASS", 0x8000000000000003LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_REJECT", 0x8000000000000004LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_RESCUE", 0x8000000000000005LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL_ON", 0xc000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL_CPU_MASK", 0xffffffffLLU }, + { "scx_dsq_iter_flags", "SCX_DSQ_ITER_REV", 0x10000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_SLICE", 0x40000000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_VTIME", 0x80000000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_USER_FLAGS", 0x10000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_ALL_FLAGS", 0xc0010000LLU }, + { "scx_dsq_lnode_flags", "SCX_DSQ_LNODE_ITER_CURSOR", 0x1LLU }, + { "scx_dsq_lnode_flags", "__SCX_DSQ_LNODE_PRIV_SHIFT", 0x10LLU }, + { "scx_enable_state", "SCX_ENABLING", 0x0LLU }, + { "scx_enable_state", "SCX_ENABLED", 0x1LLU }, + { "scx_enable_state", "SCX_DISABLING", 0x2LLU }, + { "scx_enable_state", "SCX_DISABLED", 0x3LLU }, + { "scx_enq_flags", "SCX_ENQ_WAKEUP", 0x1LLU }, + { "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU }, + { "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU }, + { "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU }, + { "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU }, + { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, + { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU }, + { "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_NESTED", 0x400000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_GDSQ_FALLBACK", 0x800000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_IGNORE_CAPS", 0x1000000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_APPLY_SLICE", 0x2000000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_SLICE_DFL", 0x4000000000000000LLU }, + { "scx_ent_dsq_flags", "SCX_TASK_DSQ_ON_PRIQ", 0x1LLU }, + { "scx_ent_flags", "SCX_TASK_QUEUED", 0x1LLU }, + { "scx_ent_flags", "SCX_TASK_IN_CUSTODY", 0x2LLU }, + { "scx_ent_flags", "SCX_TASK_RESET_RUNNABLE_AT", 0x4LLU }, + { "scx_ent_flags", "SCX_TASK_DEQD_FOR_SLEEP", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU }, + { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, + { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, + { "scx_ent_flags", "SCX_TASK_NONE", 0x0LLU }, + { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x100LLU }, + { "scx_ent_flags", "SCX_TASK_INIT", 0x200LLU }, + { "scx_ent_flags", "SCX_TASK_READY", 0x300LLU }, + { "scx_ent_flags", "SCX_TASK_ENABLED", 0x400LLU }, + { "scx_ent_flags", "SCX_TASK_DEAD", 0x500LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xcLLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x3LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x7000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, + { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, + { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, + { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, + { "scx_exit_code", "SCX_ECODE_ACT_RESTART", 0x1000000000000LLU }, + { "scx_exit_flags", "SCX_EFLAG_INITIALIZED", 0x1LLU }, + { "scx_exit_kind", "SCX_EXIT_NONE", 0x0LLU }, + { "scx_exit_kind", "SCX_EXIT_DONE", 0x1LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG", 0x40LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG_BPF", 0x41LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG_KERN", 0x42LLU }, + { "scx_exit_kind", "SCX_EXIT_SYSRQ", 0x43LLU }, + { "scx_exit_kind", "SCX_EXIT_PARENT", 0x44LLU }, + { "scx_exit_kind", "SCX_EXIT_PARENT_KILL", 0x45LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR", 0x400LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_BPF", 0x401LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_STALL", 0x402LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_REENQ", 0x403LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_RESCUE", 0x404LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_UNLOCKED", 0x1LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_INIT_CIDS", 0x2LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_CPU_RELEASE", 0x4LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_DISPATCH", 0x8LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_ENQUEUE", 0x10LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_SELECT_CPU", 0x20LLU }, + { "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU }, + { "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU }, + { "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU }, + { "scx_opi", "SCX_OPI_BEGIN", 0x0LLU }, + { "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU }, + { "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU }, + { "scx_opi", "SCX_OPI_CPU_HOTPLUG_BEGIN", 0x21LLU }, + { "scx_opi", "SCX_OPI_CPU_HOTPLUG_END", 0x23LLU }, + { "scx_opi", "SCX_OPI_END", 0x23LLU }, + { "scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE", 0x1LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_LAST", 0x2LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_EXITING", 0x4LLU }, + { "scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL", 0x8LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED", 0x10LLU }, + { "scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP", 0x20LLU }, + { "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU }, + { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, + { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU }, + { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, + { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, + { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, + { "scx_ops_state", "SCX_OPSS_QUEUEING", 0x1LLU }, + { "scx_ops_state", "SCX_OPSS_QUEUED", 0x2LLU }, + { "scx_ops_state", "SCX_OPSS_DISPATCHING", 0x3LLU }, + { "scx_ops_state", "SCX_OPSS_QSEQ_SHIFT", 0x2LLU }, + { "scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE", 0x1LLU }, + { "scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE", 0x2LLU }, + { "scx_public_consts", "SCX_OPS_NAME_LEN", 0x80LLU }, + { "scx_public_consts", "SCX_SLICE_DFL", 0x1312d00LLU }, + { "scx_public_consts", "SCX_SLICE_BYPASS", 0x4c4b40LLU }, + { "scx_public_consts", "SCX_SLICE_INF", 0xffffffffffffffffLLU }, + { "scx_reenq_flags", "SCX_REENQ_ANY", 0x1LLU }, + { "scx_reenq_flags", "SCX_REENQ_CAP_REVOKE", 0x2LLU }, + { "scx_reenq_flags", "__SCX_REENQ_FILTER_MASK", 0xffffLLU }, + { "scx_reenq_flags", "__SCX_REENQ_USER_MASK", 0x1LLU }, + { "scx_reenq_flags", "SCX_REENQ_TSR_RQ_OPEN", 0x100000000LLU }, + { "scx_reenq_flags", "SCX_REENQ_TSR_NOT_FIRST", 0x200000000LLU }, + { "scx_reenq_flags", "__SCX_REENQ_TSR_MASK", 0xf00000000LLU }, + { "scx_rq_flags", "SCX_RQ_ONLINE", 0x1LLU }, + { "scx_rq_flags", "SCX_RQ_CAN_STOP_TICK", 0x2LLU }, + { "scx_rq_flags", "SCX_RQ_CLK_VALID", 0x20LLU }, + { "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU }, + { "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU }, + { "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU }, + { "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU }, + { "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU }, + { "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_DUR_BITS", 0x2bLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_BITS", 0x14LLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_DUR_MASK", 0x7ffffffffffLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_SHIFT", 0x2bLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_MASK", 0xfffffLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_PENDING", 0x8000000000000000LLU }, + { "scx_tg_flags", "SCX_TG_ONLINE", 0x1LLU }, + { "scx_tg_flags", "SCX_TG_INITED", 0x2LLU }, + { "scx_tg_flags", "SCX_TG_SUB_INIT", 0x4LLU }, + { "scx_wake_flags", "SCX_WAKE_FORK", 0x4LLU }, + { "scx_wake_flags", "SCX_WAKE_TTWU", 0x8LLU }, + { "scx_wake_flags", "SCX_WAKE_SYNC", 0x10LLU }, +}; + +#endif /* __ENUMS_ABI_AUTOGEN_H__ */ From 9e8581a090c02ffa35e8439b90b024956a735de9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 18 Aug 2026 09:48:40 -1000 Subject: [PATCH 05/12] sched_ext: Sync common and compat headers from the scx repo Sync common.bpf.h, compat.bpf.h and compat.h with the scx repo, which accumulated the following: - __COMPAT_read_enum() can now recover 64-bit scx enum values from kernel BTF generated without BTF_KIND_ENUM64 support (pahole < 1.24 or --skip_encoding_btf_enum64, e.g. COS/GKE kernels), substituting values from the build-time vmlinux.h cross-checked against the low 32 bits the kernel does provide. - is_migration_disabled() no longer assumes the BPF prolog always disables migration. Since 8e4f0b1ebcf2 ("bpf: use rcu_read_lock_dont_migrate() for trampoline.c") the prolog only does so under CONFIG_PREEMPT_RCU, so the old current-task test under-reported on v6.18+ !PREEMPT_RCU kernels. A runtime probe on bpf_scx_reg() handles older kernels with backported trampoline behavior. - __COMPAT_scx_bpf_dsq_peek() is gated behind kernel v7.1 where 2f2ea7709266 ("sched_ext: Use dsq->first_task instead of list_empty() in dispatch_enqueue() FIFO-tail") fixed the kfunc spuriously returning NULL on non-empty FIFO DSQs, and the new scx_bpf_reenqueue_local_from_anywhere() provides a callable-from-anywhere reenqueue which prefers the generic scx_bpf_dsq_reenq(). Both were first posted by Gavin Guo and Changwoo Min and are picked up here with the review feedback folded in. - __COMPAT_scx_bpf_cpu_curr() and the scx_bpf_cpu_rq() declaration are restored. Schedulers built from these headers still run on pre-v6.18 kernels where scx_bpf_cpu_curr() does not resolve and the scx_bpf_cpu_rq() fallback still exists. - scx_clock_task() and scx_clock_pelt() document their stale-read behavior for remote idle CPUs under NO_HZ_IDLE. Link: https://lore.kernel.org/all/20260817143126.562923-1-changwoo@igalia.com Signed-off-by: Tejun Heo --- tools/sched_ext/include/scx/common.bpf.h | 140 +++++++++++++++++++---- tools/sched_ext/include/scx/compat.bpf.h | 68 +++++++++-- tools/sched_ext/include/scx/compat.h | 97 ++++++++++++++++ 3 files changed, 274 insertions(+), 31 deletions(-) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 979d4cabfaf9..76f5e025e107 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -48,6 +48,7 @@ extern int LINUX_KERNEL_VERSION __kconfig; extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; +extern bool CONFIG_PREEMPT_RCU __kconfig __weak; /* * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can @@ -97,6 +98,7 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; bool scx_bpf_task_running(const struct task_struct *p) __ksym; s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; +struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym __weak; struct rq *scx_bpf_locked_rq(void) __ksym; struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; @@ -527,32 +529,103 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) return (const struct cpumask *)mask; } +/* + * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls + * migrate_disable() for the current task. Recorded once by + * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during + * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!). + * + * Defaults to true (conservative). Over-reporting in is_migration_disabled() + * causes local-only dispatch, which is safe. Under-reporting can crash the + * scheduler, so we err high if the probe somehow fails to run. + */ +bool __scx_prolog_disables_migration __weak = true; + +/* + * scx_lib_init_probe - non-sleepable prolog probe. + * + * Attached to bpf_scx_reg(), the .reg callback in bpf_sched_ext_ops + * (kernel/sched/ext.c). The kernel's struct_ops machinery invokes + * bpf_scx_reg when userspace creates the scheduler link, before + * ops.init() fires. Its address is taken in the vtable, so the symbol + * is non-inlinable and has been stable since introduction. + * + * Entering via fentry runs us through __bpf_prog_enter -- the + * non-sleepable prolog that consumers of is_migration_disabled() live + * under. + * + * Loud warning: the prolog adds at most 1 to migration_disabled. + * Reading > 1 means something upstream in the + * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration + * before the prolog ran, invalidating the probe; audit and adjust. + */ +SEC("fentry/bpf_scx_reg") __weak +int scx_lib_init_probe(void *ctx) +{ + if (bpf_core_field_exists(((struct task_struct *)0)->migration_disabled)) { + const struct task_struct *p = bpf_get_current_task_btf(); + unsigned int md = p->migration_disabled; + + if (md > 1) + bpf_printk("scx_lib_init_probe: unexpected migration_disabled=%u " + "upstream of BPF prolog; probe result unreliable", + md); + + __scx_prolog_disables_migration = md > 0; + } + return 0; +} + /* * Return true if task @p cannot migrate to a different CPU, false * otherwise. + * + * IMPORTANT: designed for NON-SLEEPABLE BPF contexts only. Sleepable + * contexts (BPF_STRUCT_OPS_SLEEPABLE, SEC("syscall"), + * SEC("fentry.s/...")) enter via __bpf_prog_enter_sleepable() or + * __bpf_prog_enter_sleepable_recur(), both of which unconditionally + * call migrate_disable(); this helper can yield a false negative for + * p == current there, which can crash the scheduler. */ static inline bool is_migration_disabled(const struct task_struct *p) { /* - * Testing p->migration_disabled in a BPF code is tricky because the - * migration is _always_ disabled while running the BPF code. - * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) for BPF - * code execution disable and re-enable the migration of the current - * task, respectively. So, the _current_ task of the sched_ext ops is - * always migration-disabled. Moreover, p->migration_disabled could be - * two or greater when a sched_ext ops BPF code (e.g., ops.tick) is - * executed in the middle of the other BPF code execution. + * Testing p->migration_disabled in BPF is tricky because the BPF prolog + * (__bpf_prog_enter) may call migrate_disable() for the current task, + * making migration_disabled == 1 even for tasks that are not truly + * migration-disabled. * - * Therefore, we should decide that the _current_ task is - * migration-disabled only when its migration_disabled count is greater - * than one. In other words, when p->migration_disabled == 1, there is - * an ambiguity, so we should check if @p is the current task or not. + * Since commit 8e4f0b1ebcf2 ("bpf: use rcu_read_lock_dont_migrate() for + * trampoline.c"), the BPF prolog calls migrate_disable() only when + * CONFIG_PREEMPT_RCU is enabled. Two fast paths cover the common cases: + * + * 1) CONFIG_PREEMPT_RCU: prolog always calls migrate_disable(), so + * migration_disabled == 1 for the current task is ambiguous. + * Disambiguate by checking p == current. + * + * 2) v6.18+ without CONFIG_PREEMPT_RCU: prolog never calls + * migrate_disable(), so migration_disabled == 1 is unambiguously + * a real migrate_disable() call. + * + * A slow path handles pre-v6.18 kernels without CONFIG_PREEMPT_RCU, + * where the prolog historically called migrate_disable() unconditionally + * but a cherry-picked downstream kernel may not. The runtime-probed flag + * __scx_prolog_disables_migration (set by scx_lib_init_probe) distinguishes + * the two cases without relying on the kernel version alone. */ if (bpf_core_field_exists(p->migration_disabled)) { - if (p->migration_disabled == 1) - return bpf_get_current_task_btf() != p; - else - return p->migration_disabled; + if (p->migration_disabled == 1) { + /* Fast path: prolog always disables migration */ + if (CONFIG_PREEMPT_RCU) + return bpf_get_current_task_btf() != p; + /* Fast path: prolog never disables migration */ + if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(6, 18, 0)) + return true; + /* Slow path: pre-v6.18, !PREEMPT_RCU - use runtime flag */ + return __scx_prolog_disables_migration ? + bpf_get_current_task_btf() != p : true; + } + return p->migration_disabled; } return false; } @@ -1021,7 +1094,20 @@ static inline u64 scx_clock_task(u32 cpu) { struct rq___local *rq = get_current_rq(cpu); - /* Equivalent to the kernel's rq_clock_task(). */ + /* + * Equivalent to the kernel's rq_clock_task(): wall-clock time minus + * cumulative IRQ time (CONFIG_IRQ_TIME_ACCOUNTING) and hypervisor + * steal time (CONFIG_PARAVIRT_TIME_ACCOUNTING). Without those configs, + * it equals rq->clock. + * + * Conceptually this clock advances during idle (the idle task counts + * as a running task), but rq->clock_task is only updated on scheduling + * events. With NO_HZ_IDLE (the default), the periodic tick is stopped + * on idle CPUs, so rq->clock_task is not refreshed while a CPU is + * idle. Reading this clock for a remote idle CPU from a BPF timer + * callback returns the value from when the CPU last went idle, making + * the delta over an idle interval effectively zero. + */ return rq ? rq->clock_task : 0; } @@ -1032,9 +1118,23 @@ static inline u64 scx_clock_pelt(u32 cpu) /* * Equivalent to the kernel's rq_clock_pelt(): subtracts * lost_idle_time from clock_pelt to absorb the jump that occurs - * when clock_pelt resyncs with clock_task at idle exit. The result - * is a continuous, capacity-invariant clock safe for both task - * execution time stamping and cross-idle measurements. + * when clock_pelt resyncs with clock_task at idle exit. The intent + * is a continuous, capacity- and frequency-invariant clock that is + * frozen during idle, IRQ, and hypervisor steal. + * + * However, like scx_clock_task(), this clock has a stale-read issue + * for remote idle CPUs with NO_HZ_IDLE (the default). clock_pelt + * itself advances at wall-clock rate (hardware-clock based), but + * lost_idle_time is only updated via update_rq_clock_pelt(), which + * requires update_rq_clock() to be called. With NO_HZ_IDLE, the + * periodic tick is stopped on idle CPUs, so lost_idle_time is not + * refreshed during idle. Reading this clock for a remote idle CPU + * from a BPF timer callback therefore returns a value that drifts + * at wall-clock rate -- the same stale behaviour as scx_clock_task(). + * + * Without NO_HZ_IDLE, periodic ticks keep lost_idle_time nearly in + * sync (stale by at most one tick period, ~1 ms), so the result is + * accurate. */ return rq ? (rq->clock_pelt - rq->lost_idle_time) : 0; } diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 3ab642f92c8a..6944221f96cc 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -92,15 +92,20 @@ int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __k /* * v6.19: Introduce lockless peek API for user DSQs. + * v7.1: Fix scx_bpf_dsq_peek() spuriously returning NULL on non-empty + * FIFO DSQs (2f2ea7709266). * - * Preserve the following macro until v6.21. + * The kfunc exists from v6.19 but can return NULL for a non-empty FIFO DSQ + * before the v7.1 fix. Require kernel version >= 7.1.0 before calling it; + * otherwise fall through to the bpf_iter_scx_dsq fallback below. */ static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id) { struct task_struct *p = NULL; struct bpf_iter_scx_dsq it; - if (bpf_ksym_exists(scx_bpf_dsq_peek)) + if (bpf_ksym_exists(scx_bpf_dsq_peek) && + LINUX_KERNEL_VERSION >= KERNEL_VERSION(7, 1, 0)) return scx_bpf_dsq_peek(dsq_id); if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0)) p = bpf_iter_scx_dsq_next(&it); @@ -238,6 +243,26 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) +/* + * v6.18: Add a helper to retrieve the current task running on a CPU. + * + * The kernel tree dropped this helper and scx_bpf_cpu_rq(), but schedulers in + * this tree still support pre-v6.18 kernels where scx_bpf_cpu_curr() doesn't + * resolve and the scx_bpf_cpu_rq() fallback still exists. Keep it until + * pre-v6.18 kernels fall out of the support window. + */ +static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) +{ + struct rq *rq; + + if (bpf_ksym_exists(scx_bpf_cpu_curr)) + return scx_bpf_cpu_curr(cpu); + + rq = scx_bpf_cpu_rq(cpu); + + return rq ? rq->curr : NULL; +} + /* * v6.19: To work around BPF maximum parameter limit, the following kfuncs are * replaced with variants that pack scalar arguments in a struct. Wrappers are @@ -378,6 +403,17 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) p->scx.dsq_vtime = vtime; } +/* + * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This + * will eventually deprecate scx_bpf_reenqueue_local(). + */ +void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; + +static inline bool __COMPAT_has_generic_reenq(void) +{ + return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); +} + /* * v6.19: The new void variant can be called from anywhere while the older v1 * variant can only be called from ops.cpu_release(). The double ___ prefixes on @@ -395,21 +431,31 @@ static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void) static inline void scx_bpf_reenqueue_local(void) { - if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) + if (__COMPAT_has_generic_reenq()) + scx_bpf_dsq_reenq___compat(SCX_DSQ_LOCAL, 0); + else if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) scx_bpf_reenqueue_local___v2___compat(); else scx_bpf_reenqueue_local___v1(); } -/* - * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This - * will eventually deprecate scx_bpf_reenqueue_local(). - */ -void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; - -static inline bool __COMPAT_has_generic_reenq(void) +static inline int scx_bpf_reenqueue_local_from_anywhere(void) { - return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); + /* + * The generic reenq kfunc and the v2 reenqueue-local variant can both be + * called from anywhere; v1 cannot. Test each ksym in its own branch with a + * distinct call: combining them with || would fold into a bitwise OR of the + * two ksym addresses, which the verifier rejects. + */ + if (__COMPAT_has_generic_reenq()) { + scx_bpf_dsq_reenq___compat(SCX_DSQ_LOCAL, 0); + return 0; + } + if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) { + scx_bpf_reenqueue_local___v2___compat(); + return 0; + } + return -EOPNOTSUPP; } static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index d2e4384df5af..7c12df45fdba 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -10,9 +10,14 @@ #include #include #include +#include +#include #include +#include #include +#include "enums_abi.autogen.h" + struct btf *__COMPAT_vmlinux_btf __attribute__((weak)); static inline void __COMPAT_load_vmlinux_btf(void) @@ -23,6 +28,85 @@ static inline void __COMPAT_load_vmlinux_btf(void) } } +/* + * Recover the true value of a 64-bit enum enumerator whose kernel BTF entry + * was truncated to its low 32 bits. + * + * Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode + * 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only + * carry the low 32 bits. This happens with pahole < 1.24, which predates + * ENUM64, and with pahole passing --skip_encoding_btf_enum64 (e.g. Google's + * Container-Optimized OS / GKE kernels deliberately pass it for backward + * compatibility with older BTF consumers). The high bits + * can't be recovered from kernel BTF, so substitute the value from the + * vmlinux.h this tree was built against, cross-checked against the low 32 + * bits the kernel did provide. + * + * Note that this is a best-effort recovery, not a ground truth. The + * substitution assumes the running kernel agrees with this tree's vmlinux.h + * on the high 32 bits, but only the low 32 bits can actually be verified. + * The cross-check is vacuous for enumerators whose value has no low bits + * set (e.g. SCX_DSQ_FLAG_BUILTIN, __SCX_ENQ_INTERNAL_MASK, + * SCX_ENQ_CLEAR_OPSS, SCX_ECODE_*): their lo32 is 0 and matches anything, + * so those substitutions rest entirely on the high bits never moving. An + * enumerator missing from the table (a kernel newer than this tree's + * vmlinux.h, or a stale autogen table) can't be recovered at all. If a + * substitution is ever wrong, the scheduler operates on bogus values (e.g. + * dispatching to nonexistent DSQ ids or silently dropping flags) and can + * wildly malfunction, which is why the mismatch and table-miss paths refuse + * instead of guessing. + */ +static inline bool __COMPAT_recover_truncated_enum64(const char *type, + const char *name, + u32 lo32, u64 *v) +{ + static bool warned; + size_t i; + + for (i = 0; i < sizeof(__scx_enum_abi_vals) / sizeof(__scx_enum_abi_vals[0]); i++) { + const struct __scx_enum_abi_val *e = &__scx_enum_abi_vals[i]; + + if (strcmp(e->type, type) || strcmp(e->name, name)) + continue; + + if (e->val <= (u64)UINT32_MAX) { + *v = lo32; + return true; + } + + if ((u32)e->val != lo32) { + fprintf(stderr, "ERROR: kernel BTF value of %s::%s (0x%x) doesn't match the low 32 bits of the vmlinux.h value (0x%llx); refusing to substitute\n", + type, name, lo32, (unsigned long long)e->val); + return false; + } + + if (!warned) { + fprintf(stderr, + "WARNING: kernel BTF lacks BTF_KIND_ENUM64 encoding (generated by\n" + "WARNING: pahole < 1.24 or with --skip_encoding_btf_enum64), so 64-bit\n" + "WARNING: scx enum values are truncated to their low 32 bits in kernel\n" + "WARNING: BTF. Substituting the full 64-bit values from the vmlinux.h\n" + "WARNING: this binary was built against, cross-checked against the low\n" + "WARNING: 32 bits the kernel does provide. The high 32 bits cannot be\n" + "WARNING: verified: if the running kernel's actual values differ from\n" + "WARNING: the build-time vmlinux.h (e.g. an enum that moved in a newer\n" + "WARNING: kernel), the scheduler will operate on bogus values, such as\n" + "WARNING: dispatching to nonexistent DSQ ids, and can wildly malfunction.\n"); + warned = true; + } + *v = e->val; + return true; + } + + /* + * Unknown enumerator (likely a stale autogen table). Fail + * pessimistically to avoid returning an invalid value. + */ + fprintf(stderr, "ERROR: kernel BTF truncates 64-bit enum %s::%s to 0x%x; 64-bit variant not found in vmlinux.h\n", + type, name, lo32); + return false; +} + static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v) { const struct btf_type *t; @@ -46,6 +130,19 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { + /* + * Try to recover a 64-bit enum from an 8-byte + * BTF_KIND_ENUM that was encoded without ENUM64 + * support (old pahole or + * --skip_encoding_btf_enum64). Only scx_* + * types are covered by the substitution table; + * non-scx types fall through to the raw value + * so this generic utility keeps working for + * them. + */ + if (t->size == 8 && !strncmp(type, "scx_", 4)) + return __COMPAT_recover_truncated_enum64(type, name, + (u32)e[i].val, v); *v = e[i].val; return true; } From cca061dccf563907061766191b2ce3f66b7c285a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 21 Aug 2026 09:05:52 -1000 Subject: [PATCH 06/12] sched_ext: Fix spurious aborts in scx_bpf_dsq_move() on ownership change races scx_dsq_move() verifies that the task belongs to the calling scheduler before taking any locks and aborts the scheduler on mismatch. The task can lose the sched association at any point: It can run and fully exit, which clears the association, or get rehomed to a different sub-sched. Both are benign races, but the early ownership check escalates them into scheduler aborts. Move the ownership check below the cursor-lost check. Every ownership change dequeues the task first, so a task that is still on the iterated DSQ under the lock while owned elsewhere indicates a genuine violation and should abort. Also fix two stale comments still referencing sched_ext_free(), which has been renamed to sched_ext_dead(). Fixes: bb4d9fd55158 ("sched_ext: scx_dsq_move() should validate the task belongs to the right scheduler") Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b646711a45fe..c539d15cda63 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7694,7 +7694,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled + * leaving as sched_ext_dead() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * @@ -7786,7 +7786,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through + * transitions here are synchronized against sched_ext_dead() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); @@ -9004,12 +9004,6 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - /* * Can be called from either ops.dispatch() holding the dispatched rq's * lock or any context where no rq lock is held. If latter, lock @p's @@ -9041,6 +9035,17 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, goto out; } + /* + * @p has been on $src_dsq and can't move anymore. If @p is not on @sch, + * the caller didn't have authority over @p at the time of the call. + */ + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + raw_spin_unlock(&src_dsq->lock); + goto out; + } + /* @p is still on $src_dsq and stable, determine the destination */ dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); From 6586705bc2dc06908309bf65d79efa53a347c9f0 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 24 Aug 2026 21:21:16 +0800 Subject: [PATCH 07/12] docs/sched_ext: document that cgroup CPU knobs are scheduler-dependent The scheduler core communicates the initial cpu controller settings to the BPF scheduler through ops.cgroup_init() and reports subsequent changes through the corresponding ops.cgroup_set_*() callbacks. Whether and how a knob takes effect is up to the loaded scheduler: it may implement the corresponding callback partially or not at all, so cpu.max, cpu.weight and friends can silently have no effect. Document this in the basics section of sched-ext.rst. Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 35b550671ca7..794ae80b3ba3 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -242,6 +242,21 @@ optional. The following modified excerpt is from .name = "simple", }; +Scheduler-Dependent Knobs +------------------------- + +The fair-class scheduler enforces CPU controller settings such as +``cpu.max``, ``cpu.weight`` and ``cpu.idle``. For sched_ext tasks, the +scheduler core communicates these settings to the BPF scheduler +through ``ops.cgroup_init()`` and reports subsequent changes through +the corresponding ``ops.cgroup_set_*()`` callbacks. Similarly, per-task +nice changes are converted to weights and reported through +``ops.set_weight()``. + +Each BPF scheduler is responsible for implementing the scheduling +semantics of these settings and may choose to ignore them. Consult the +loaded scheduler's documentation before relying on these controls. + Dispatch Queues --------------- From 23761359861ca4bb087540937dfea8b0716914c2 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 27 Aug 2026 16:07:36 +0800 Subject: [PATCH 08/12] sched_ext: Fix timer pinning and return value in scx_central central_timerfn() re-arms the timer with a hardcoded BPF_F_TIMER_CPU_PIN flag and ignores the return value, defeating start_central_timer()'s -EINVAL fallback for kernels without the flag (<6.7): on such kernels the first tick kills the timer permanently with no diagnostic. Honor timer_pinned and check the return like the initial arm does. Fixes: 22a920209ab6 ("sched_ext: Implement tickless support") Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- tools/sched_ext/scx_central.bpf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c index 64dd60b3e922..65dae9e45400 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -299,6 +299,7 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) u64 now = scx_bpf_now(); u64 nr_to_kick = nr_queued; s32 i, curr_cpu; + int ret; curr_cpu = bpf_get_smp_processor_id(); if (timer_pinned && (curr_cpu != central_cpu)) { @@ -332,7 +333,10 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); } - bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN); + ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, + timer_pinned ? BPF_F_TIMER_CPU_PIN : 0); + if (ret) + scx_bpf_error("bpf_timer_start failed (%d)", ret); __sync_fetch_and_add(&nr_timers, 1); return 0; } From b6ee92d7f7f0498d1f776d0b125a2f6bcedf0891 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 27 Aug 2026 16:07:37 +0800 Subject: [PATCH 09/12] sched_ext: Fix vtime delta loss in scx_flatcg cgroup migration fcg_cgroup_move() lost the signed vtime offset across cgroup migration in the mechanical conversion to time helpers: time_delta() clamps negative deltas to 0, so a queued task (whose dsq_vtime is normally behind the source frontier) loses its accumulated vtime credit and lands exactly at the destination frontier instead of keeping its relative position. Restore the wrapping signed subtraction. Fixes: 62addc6dbf36 ("sched_ext: Use time helpers in BPF schedulers") Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- tools/sched_ext/scx_flatcg.bpf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 64cf4dd964d6..454ebb820c5e 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -937,7 +937,7 @@ void BPF_STRUCT_OPS(fcg_cgroup_move, struct task_struct *p, if (!(from_cgc = find_cgrp_ctx(from)) || !(to_cgc = find_cgrp_ctx(to))) return; - delta = time_delta(p->scx.dsq_vtime, from_cgc->tvtime_now); + delta = (s64)(p->scx.dsq_vtime - from_cgc->tvtime_now); scx_bpf_task_set_dsq_vtime(p, to_cgc->tvtime_now + delta); } From 84590dbb9f3519e865ee8396494ac7186b625fef Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 27 Aug 2026 16:07:38 +0800 Subject: [PATCH 10/12] sched_ext: Check bpf_timer_start return values in scx_qmap monitor_timerfn(), lowpri_timerfn() and round_robin_timerfn() ignore bpf_timer_start()'s return value: a failed re-arm silently stops the periodic heartbeat, starving every task parked in LOWPRI_DSQ (lowpri) or freezing cid rotation (round-robin). Check the returns and raise scx_bpf_error(), matching the init paths. Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- tools/sched_ext/scx_qmap.bpf.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 5bb8b90a275a..9f6e61d7ca07 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1246,7 +1246,8 @@ static int monitor_timerfn(void *map, int *key, struct bpf_timer *timer) scx_read_event(&events, SCX_EV_BYPASS_ACTIVATE)); } - bpf_timer_start(timer, ONE_SEC_IN_NS, 0); + if (bpf_timer_start(timer, ONE_SEC_IN_NS, 0)) + scx_bpf_error("failed to re-arm stats timer"); return 0; } @@ -1268,7 +1269,8 @@ struct { static int lowpri_timerfn(void *map, int *key, struct bpf_timer *timer) { scx_bpf_dsq_reenq(LOWPRI_DSQ, 0); - bpf_timer_start(timer, LOWPRI_INTV_NS, 0); + if (bpf_timer_start(timer, LOWPRI_INTV_NS, 0)) + scx_bpf_error("failed to re-arm lowpri timer"); return 0; } @@ -1747,7 +1749,8 @@ static void rr_advance(void) static int round_robin_timerfn(void *map, int *key, struct bpf_timer *timer) { rr_advance(); - bpf_timer_start(timer, round_robin_ns, 0); + if (bpf_timer_start(timer, round_robin_ns, 0)) + scx_bpf_error("failed to re-arm round-robin timer"); return 0; } From 4881a13521886076e8d6d677f274320dcf66fea4 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 27 Aug 2026 17:14:11 +0800 Subject: [PATCH 11/12] sched_ext: Fix several comment issues Fix several comment issues found during review: __setschduler_prio() -> __setscheduler_class() scx_iter_scx_dsq_new() -> bpf_iter_scx_dsq_new() scx_next_task_scx() -> set_next_task_scx() Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 +++++----- kernel/sched/ext/internal.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c539d15cda63..76a3f4ea237c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -876,9 +876,9 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) * unloading. The init_tasks ("swappers") should be excluded * from the iteration because: * - * - It's unsafe to use __setschduler_prio() on an init_task to - * determine the sched_class to use as it won't preserve its - * idle_sched_class. + * - It's unsafe to use __setscheduler_class() on an init_task + * to determine the sched_class to use as it won't preserve + * its idle_sched_class. * * - ops.init/exit_task() can easily be confused if called with * init_tasks as they, e.g., share PID 0. @@ -5514,7 +5514,7 @@ static const struct kset_uevent_ops scx_uevent_ops = { }; /* - * Used by sched_fork() and __setscheduler_prio() to pick the matching + * Used by sched_fork() and __setscheduler_class() to pick the matching * sched_class. dl/rt are already handled. */ bool task_should_scx(int policy) @@ -9771,7 +9771,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo scx_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 53e136a47924..0967b99a4948 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -442,7 +442,7 @@ struct sched_ext_ops { * * Note that this callback may be called from a CPU other than the * one the task is going to run on. This can happen when a task - * property is changed (i.e., affinity), since scx_next_task_scx(), + * property is changed (i.e., affinity), since set_next_task_scx(), * which triggers this callback, may run on a CPU different from * the task's assigned CPU. * From 068e5a0bc57e57d24cbf38def29cc5fb4db9a0df Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Tue, 25 Aug 2026 13:50:53 +0800 Subject: [PATCH 12/12] sched_ext: Fix missing @slice and @vtime descriptions in finish_dispatch() kernel-doc Commit 13f1eae3b662 ("sched_ext: Synchronize slice and dsq_vtime writes") added the slice and vtime parameters to finish_dispatch() but did not update its kernel-doc, which produces warnings: Warning: function parameter 'slice' not described in 'finish_dispatch' Warning: function parameter 'vtime' not described in 'finish_dispatch' Describe both parameters using the same wording as dispatch_to_local_dsq(), which receives the same values. Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 76a3f4ea237c..713aa26b2828 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2806,6 +2806,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched * @dsq_id: destination DSQ ID + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * Dispatching to local DSQs may need to wait for queueing to complete or