From bba2c3615bd6cfee7456d1130f2e6b01b3f4e9ba Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 05:32:54 -1000 Subject: [PATCH 1/3] sched_ext: Move sources under kernel/sched/ext/ The sched_ext sources had grown to ten ext* files directly under kernel/sched/. Move them into a new kernel/sched/ext/ subdirectory and drop the now-redundant ext_ prefix. ext.c/h keep their names. kernel/sched/ext.{c,h} -> kernel/sched/ext/ext.{c,h} kernel/sched/ext_internal.h -> kernel/sched/ext/internal.h kernel/sched/ext_types.h -> kernel/sched/ext/types.h kernel/sched/ext_idle.{c,h} -> kernel/sched/ext/idle.{c,h} kernel/sched/ext_cid.{c,h} -> kernel/sched/ext/cid.{c,h} kernel/sched/ext_arena.{c,h} -> kernel/sched/ext/arena.{c,h} The include paths in build_policy.c and sched.h, the MAINTAINERS glob, and a few documentation and comment references are updated to match. No code or symbol changes. Suggested-by: Linus Torvalds Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 8 ++++---- MAINTAINERS | 2 +- kernel/sched/build_policy.c | 18 +++++++++--------- kernel/sched/{ext_arena.c => ext/arena.c} | 0 kernel/sched/{ext_arena.h => ext/arena.h} | 0 kernel/sched/{ext_cid.c => ext/cid.c} | 2 +- kernel/sched/{ext_cid.h => ext/cid.h} | 2 +- kernel/sched/{ => ext}/ext.c | 4 ++-- kernel/sched/{ => ext}/ext.h | 0 kernel/sched/{ext_idle.c => ext/idle.c} | 0 kernel/sched/{ext_idle.h => ext/idle.h} | 0 .../sched/{ext_internal.h => ext/internal.h} | 0 kernel/sched/{ext_types.h => ext/types.h} | 0 kernel/sched/sched.h | 2 +- tools/sched_ext/include/scx/cid.bpf.h | 6 +++--- 15 files changed, 22 insertions(+), 22 deletions(-) rename kernel/sched/{ext_arena.c => ext/arena.c} (100%) rename kernel/sched/{ext_arena.h => ext/arena.h} (100%) rename kernel/sched/{ext_cid.c => ext/cid.c} (99%) rename kernel/sched/{ext_cid.h => ext/cid.h} (99%) rename kernel/sched/{ => ext}/ext.c (99%) rename kernel/sched/{ => ext}/ext.h (100%) rename kernel/sched/{ext_idle.c => ext/idle.c} (100%) rename kernel/sched/{ext_idle.h => ext/idle.h} (100%) rename kernel/sched/{ext_internal.h => ext/internal.h} (100%) rename kernel/sched/{ext_types.h => ext/types.h} (100%) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index c4f59c08d8a4..4b1ffd03f516 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -114,7 +114,7 @@ counters. Each counter occupies one ``name value`` line: SCX_EV_INSERT_NOT_OWNED 0 SCX_EV_SUB_BYPASS_DISPATCH 0 -The counters are described in ``kernel/sched/ext_internal.h``; briefly: +The counters are described in ``kernel/sched/ext/internal.h``; briefly: * ``SCX_EV_SELECT_CPU_FALLBACK``: ops.select_cpu() returned a CPU unusable by the task and the core scheduler silently picked a fallback CPU. @@ -496,11 +496,11 @@ Where to Look * ``include/linux/sched/ext.h`` defines the core data structures, ops table and constants. -* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. +* ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers. The functions prefixed with ``scx_bpf_`` can be called from the BPF scheduler. -* ``kernel/sched/ext_idle.c`` contains the built-in idle CPU selection policy. +* ``kernel/sched/ext/idle.c`` contains the built-in idle CPU selection policy. * ``tools/sched_ext/`` hosts example BPF scheduler implementations. @@ -557,7 +557,7 @@ ABI Instability The APIs provided by sched_ext to BPF schedulers programs have no stability guarantees. This includes the ops table callbacks and constants defined in ``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in -``kernel/sched/ext.c`` and ``kernel/sched/ext_idle.c``. +``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``. While we will attempt to provide a relatively stable API surface when possible, they are subject to change without warning between kernel diff --git a/MAINTAINERS b/MAINTAINERS index ba45953bb805..2d853aadd919 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24164,7 +24164,7 @@ S: Maintained W: https://github.com/sched-ext/scx T: git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git F: include/linux/sched/ext.h -F: kernel/sched/ext* +F: kernel/sched/ext/ F: tools/sched_ext/ F: tools/testing/selftests/sched_ext diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 067979a7b69e..d74b54f81992 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -61,15 +61,15 @@ # include # include # include -# include "ext_types.h" -# include "ext_internal.h" -# include "ext_cid.h" -# include "ext_arena.h" -# include "ext_idle.h" -# include "ext.c" -# include "ext_cid.c" -# include "ext_arena.c" -# include "ext_idle.c" +# include "ext/types.h" +# include "ext/internal.h" +# include "ext/cid.h" +# include "ext/arena.h" +# include "ext/idle.h" +# include "ext/ext.c" +# include "ext/cid.c" +# include "ext/arena.c" +# include "ext/idle.c" #endif #include "syscalls.c" diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext/arena.c similarity index 100% rename from kernel/sched/ext_arena.c rename to kernel/sched/ext/arena.c diff --git a/kernel/sched/ext_arena.h b/kernel/sched/ext/arena.h similarity index 100% rename from kernel/sched/ext_arena.h rename to kernel/sched/ext/arena.h diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext/cid.c similarity index 99% rename from kernel/sched/ext_cid.c rename to kernel/sched/ext/cid.c index 66944a7ef79d..aeaea88f34c5 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext/cid.c @@ -71,7 +71,7 @@ static s32 scx_cid_arrays_alloc(void) * scx_cid_init - build the cid mapping * @sch: the scx_sched being initialized; used as the scx_error() target * - * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by + * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by * intersection at each level (parent_scratch & this_level_mask), which keeps * containment correct by construction and naturally splits a physical LLC * straddling two NUMA nodes into two LLC units. The caller must hold diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext/cid.h similarity index 99% rename from kernel/sched/ext_cid.h rename to kernel/sched/ext/cid.h index 5745e5785e89..6e657fd147b0 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext/cid.h @@ -43,7 +43,7 @@ struct scx_sched; * possible-but-not-online cpus and carries all-(-1) topo info (see * scx_cid_topo); callers detect it via the -1 sentinels. * - * See the comment above the table definitions in ext_cid.c for the + * See the comment above the table definitions in cid.c for the * memory-ordering and visibility contract. */ extern s16 *scx_cid_to_cpu_tbl; diff --git a/kernel/sched/ext.c b/kernel/sched/ext/ext.c similarity index 99% rename from kernel/sched/ext.c rename to kernel/sched/ext/ext.c index 0db6fa2daea3..00fe6cc6d7e2 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext/ext.c @@ -513,7 +513,7 @@ do { \ } while (0) /* - * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so + * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. */ DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); @@ -608,7 +608,7 @@ do { \ * @cpumask: new cpumask * * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller + * scratch in cid.c and dispatch through the ops_cid union view. Caller * must hold @rq's rq lock so this_cpu_ptr is stable across the call. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, diff --git a/kernel/sched/ext.h b/kernel/sched/ext/ext.h similarity index 100% rename from kernel/sched/ext.h rename to kernel/sched/ext/ext.h diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext/idle.c similarity index 100% rename from kernel/sched/ext_idle.c rename to kernel/sched/ext/idle.c diff --git a/kernel/sched/ext_idle.h b/kernel/sched/ext/idle.h similarity index 100% rename from kernel/sched/ext_idle.h rename to kernel/sched/ext/idle.h diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext/internal.h similarity index 100% rename from kernel/sched/ext_internal.h rename to kernel/sched/ext/internal.h diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext/types.h similarity index 100% rename from kernel/sched/ext_types.h rename to kernel/sched/ext/types.h diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c7c2dea65edd..56acf502ba26 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4211,6 +4211,6 @@ DEFINE_CLASS(sched_change, struct sched_change_ctx *, DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) -#include "ext.h" +#include "ext/ext.h" #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 9d89bb57e201..db247e42fb45 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * BPF-side helpers for cids and cmasks. See kernel/sched/ext_cid.h for the + * BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the * authoritative layout and semantics. The BPF-side helpers use the cmask_* * naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is * redundant in BPF code. Atomics use __sync_val_compare_and_swap and every @@ -33,7 +33,7 @@ #endif /* - * Mirrors SCX_CMASK_NR_WORDS in kernel/sched/ext_types.h. The u64 cast keeps + * Mirrors SCX_CMASK_NR_WORDS in kernel/sched/ext/types.h. The u64 cast keeps * the +63 from wrapping when @nr_cids is near U32_MAX, so cmask_reframe() * bounds-checking the result against alloc_words catches the overflow instead * of seeing a small value. @@ -281,7 +281,7 @@ static __always_inline void cmask_zero(struct scx_cmask __arena *m) /* * BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_* - * enum in ext_cid.c, which is exported via BTF and reachable through + * enum in ext/cid.c, which is exported via BTF and reachable through * vmlinux.h. */ enum { From 3cd1f76be638b7386201171e7bb4c88095774dd5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 07:29:39 -1000 Subject: [PATCH 2/3] sched_ext: Make kernel/sched/ext/ sources self-contained for clangd The sources under kernel/sched/ext/ build as a single translation unit: build_policy.c includes the source files and headers. An LSP/clangd editor parses each as a standalone unit, sees no types, and reports a flood of errors. Give each header its dependencies and include guard, and have each source include the headers it uses. ext.c, arena.c and the ext headers now parse clean standalone. idle.c and cid.c still reference a few macros and helpers defined in ext.c. The next patch moves those to shared headers. Suggested-by: Peter Zijlstra Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/arena.c | 4 ++++ kernel/sched/ext/arena.h | 2 ++ kernel/sched/ext/cid.c | 3 +++ kernel/sched/ext/cid.h | 2 ++ kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/idle.c | 3 +++ kernel/sched/ext/idle.h | 4 ++++ kernel/sched/ext/internal.h | 8 ++++++++ kernel/sched/ext/types.h | 6 ++++++ 9 files changed, 45 insertions(+) diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c index 493c2424f842..5783694ec21d 100644 --- a/kernel/sched/ext/arena.c +++ b/kernel/sched/ext/arena.c @@ -15,6 +15,10 @@ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. * Copyright (c) 2026 Tejun Heo */ +#include + +#include "internal.h" +#include "arena.h" enum scx_arena_consts { SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ diff --git a/kernel/sched/ext/arena.h b/kernel/sched/ext/arena.h index 4f3610160102..c378ae5fbc02 100644 --- a/kernel/sched/ext/arena.h +++ b/kernel/sched/ext/arena.h @@ -8,6 +8,8 @@ #ifndef _KERNEL_SCHED_EXT_ARENA_H #define _KERNEL_SCHED_EXT_ARENA_H +#include + struct scx_sched; s32 scx_arena_pool_init(struct scx_sched *sch); diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index aeaea88f34c5..af83084ec740 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -7,6 +7,9 @@ */ #include +#include "internal.h" +#include "cid.h" + /* * cid tables. * diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 6e657fd147b0..41d0802c6af3 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -33,6 +33,8 @@ #ifndef _KERNEL_SCHED_EXT_CID_H #define _KERNEL_SCHED_EXT_CID_H +#include "internal.h" + struct scx_sched; /* diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 00fe6cc6d7e2..f3253c946764 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6,6 +6,19 @@ * Copyright (c) 2022 Tejun Heo * Copyright (c) 2022 David Vernet */ +#include +#include +#include +#include +#include +#include +#include + +#include "../pelt.h" +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "idle.h" static DEFINE_RAW_SPINLOCK(scx_sched_lock); diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 2077373d8da3..8e8c6201b7df 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -9,6 +9,9 @@ * Copyright (c) 2022 David Vernet * Copyright (c) 2024 Andrea Righi */ +#include "internal.h" +#include "cid.h" +#include "idle.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); diff --git a/kernel/sched/ext/idle.h b/kernel/sched/ext/idle.h index 8d169d3bbdf9..87a0e58f1eb7 100644 --- a/kernel/sched/ext/idle.h +++ b/kernel/sched/ext/idle.h @@ -10,7 +10,11 @@ #ifndef _KERNEL_SCHED_EXT_IDLE_H #define _KERNEL_SCHED_EXT_IDLE_H +#include + +struct cpumask; struct sched_ext_ops; +struct task_struct; extern struct btf_id_set8 scx_kfunc_ids_idle; extern struct btf_id_set8 scx_kfunc_ids_select_cpu; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b04701190b23..1f5312b3b387 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -5,6 +5,12 @@ * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. * Copyright (c) 2025 Tejun Heo */ +#ifndef _KERNEL_SCHED_EXT_INTERNAL_H +#define _KERNEL_SCHED_EXT_INTERNAL_H + +#include "../sched.h" +#include "types.h" + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) @@ -1651,3 +1657,5 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) return rcu_dereference_all(scx_root); } #endif /* CONFIG_EXT_SUB_SCHED */ + +#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 8b3527e21fca..bc74eafd43f1 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -8,6 +8,12 @@ #ifndef _KERNEL_SCHED_EXT_TYPES_H #define _KERNEL_SCHED_EXT_TYPES_H +#include +#include +#include +#include +#include + enum scx_consts { SCX_DSP_DFL_MAX_BATCH = 32, SCX_DSP_MAX_LOOPS = 32, From 4437ad129cf5b37c00a5bc9fa5989d1da4d64d07 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 07:29:39 -1000 Subject: [PATCH 3/3] sched_ext: Move shared helpers from ext.c into internal.h and cid.h idle.c and cid.c are included into build_policy.c together with ext.c and use helpers that ext.c defines. Because the helpers live in ext.c, the two files can not parse as standalone units and clangd reports errors in them. Move the helpers to the headers they belong to. The op-dispatch macros and helpers plus scx_parent() to internal.h, and scx_cpu_arg()/scx_cpu_ret() to cid.h. No functional change. idle.c and cid.c now parse clean standalone. Suggested-by: Peter Zijlstra Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.h | 21 ++++++ kernel/sched/ext/ext.c | 141 ------------------------------------ kernel/sched/ext/internal.h | 121 +++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 141 deletions(-) diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 41d0802c6af3..9c4f4b907f12 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -270,4 +270,25 @@ static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ __w &= __w - 1) +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static inline s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f3253c946764..691d53fe0f64 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -259,8 +259,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, return ret; } -#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) - static long jiffies_delta_msecs(unsigned long at, unsigned long now) { if (time_after(at, now)) @@ -275,20 +273,6 @@ static bool u32_before(u32 a, u32 b) } #ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_parent - Find the parent sched - * @sch: sched to find the parent of - * - * Returns the parent scheduler or %NULL if @sch is root. - */ -static struct scx_sched *scx_parent(struct scx_sched *sch) -{ - if (sch->level) - return sch->ancestors[sch->level - 1]; - else - return NULL; -} - /** * scx_next_descendant_pre - find the next descendant for pre-order walk * @pos: the current position (%NULL to initiate traversal) @@ -336,7 +320,6 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) rcu_assign_pointer(p->scx.sched, sch); } #else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -496,123 +479,12 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); -static inline void update_locked_rq(struct rq *rq) -{ - /* - * Check whether @rq is actually locked. This can help expose bugs - * or incorrect assumptions about the context in which a kfunc or - * callback is executed. - */ - if (rq) - lockdep_assert_rq_held(rq); - __this_cpu_write(scx_locked_rq_state, rq); -} - -/* - * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not - * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. - */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ -do { \ - struct rq *__prev_locked_rq; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ -} while (0) - /* * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. */ DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); -/* - * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form - * schedulers it resolves to the matching cid; for cpu-form it passes @cpu - * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op - * (currently only ops.select_cpu); it validates the BPF-supplied cid and - * triggers scx_error() on @sch if invalid. - */ -static s32 scx_cpu_arg(s32 cpu) -{ - if (scx_is_cid_type()) - return __scx_cpu_to_cid(cpu); - return cpu; -} - -static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) -{ - if (cpu_or_cid < 0 || !scx_is_cid_type()) - return cpu_or_cid; - return scx_cid_to_cpu(sch, cpu_or_cid); -} - -#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ -({ \ - struct rq *__prev_locked_rq; \ - __typeof__((sch)->ops.op(args)) __ret; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - __ret = (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ - __ret; \ -}) - -/* - * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments - * and records them in current->scx.kf_tasks[] for the duration of the call. A - * kfunc invoked from inside such an op can then use - * scx_kf_arg_task_ok() to verify that its task argument is one of - * those subject tasks. - * - * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - - * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's - * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. - * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. - * - * kf_tasks[] can not stack, so task-based SCX ops must not nest. The - * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants - * while a previous one is still in progress. - */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ -do { \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ -} while (0) - -#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ -({ \ - __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - __ret; \ -}) - -#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ -({ \ - __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task0; \ - current->scx.kf_tasks[1] = task1; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - current->scx.kf_tasks[1] = NULL; \ - __ret; \ -}) - /** * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task * @sch: scx_sched being invoked @@ -651,19 +523,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, current->scx.kf_tasks[0] = NULL; } -/* see SCX_CALL_OP_TASK() */ -static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, - struct task_struct *p) -{ - if (unlikely((p != current->scx.kf_tasks[0] && - p != current->scx.kf_tasks[1]))) { - scx_error(sch, "called on a task not being operated on"); - return false; - } - - return true; -} - enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1f5312b3b387..145272cb4d8a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1553,6 +1553,111 @@ static inline struct rq *scx_locked_rq(void) return __this_cpu_read(scx_locked_rq_state); } +static inline void update_locked_rq(struct rq *rq) +{ + /* + * Check whether @rq is actually locked. This can help expose bugs + * or incorrect assumptions about the context in which a kfunc or + * callback is executed. + */ + if (rq) + lockdep_assert_rq_held(rq); + __this_cpu_write(scx_locked_rq_state, rq); +} + +#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) + +/* + * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not + * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + */ +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +do { \ + struct rq *__prev_locked_rq; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ +} while (0) + +#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ +({ \ + struct rq *__prev_locked_rq; \ + __typeof__((sch)->ops.op(args)) __ret; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + __ret = (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ + __ret; \ +}) + +/* + * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments + * and records them in current->scx.kf_tasks[] for the duration of the call. A + * kfunc invoked from inside such an op can then use + * scx_kf_arg_task_ok() to verify that its task argument is one of + * those subject tasks. + * + * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - + * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's + * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. + * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. + * + * kf_tasks[] can not stack, so task-based SCX ops must not nest. The + * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants + * while a previous one is still in progress. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ +({ \ + __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ +({ \ + __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task0; \ + current->scx.kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + current->scx.kf_tasks[1] = NULL; \ + __ret; \ +}) + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, + struct task_struct *p) +{ + if (unlikely((p != current->scx.kf_tasks[0] && + p != current->scx.kf_tasks[1]))) { + scx_error(sch, "called on a task not being operated on"); + return false; + } + + return true; +} + static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) { return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & @@ -1633,6 +1738,20 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) return NULL; } + +/** + * scx_parent - Find the parent sched + * @sch: sched to find the parent of + * + * Returns the parent scheduler or %NULL if @sch is root. + */ +static inline struct scx_sched *scx_parent(struct scx_sched *sch) +{ + if (sch->level) + return sch->ancestors[sch->level - 1]; + else + return NULL; +} #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_task_sched(const struct task_struct *p) { @@ -1656,6 +1775,8 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { return rcu_dereference_all(scx_root); } + +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } #endif /* CONFIG_EXT_SUB_SCHED */ #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */