diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 2771ea4cc14a..0e97fd019994 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -93,20 +93,20 @@ scheduler has been loaded): # cat /sys/kernel/sched_ext/enable_seq 1 -Each running scheduler also exposes a per-scheduler ``events`` file under -``/sys/kernel/sched_ext//events`` that tracks diagnostic -counters. Each counter occupies one ``name value`` line: +Each running scheduler exposes an ``events`` file under its sysfs kobject +(``/sys/kernel/sched_ext/root/events`` for the root scheduler) that tracks +diagnostic counters. Each counter occupies one ``name value`` line: .. code-block:: none - # cat /sys/kernel/sched_ext/simple/events + # cat /sys/kernel/sched_ext/root/events SCX_EV_SELECT_CPU_FALLBACK 0 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE 0 SCX_EV_DISPATCH_KEEP_LAST 123 SCX_EV_ENQ_SKIP_EXITING 0 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED 0 SCX_EV_REENQ_IMMED 0 - SCX_EV_REENQ_LOCAL_REPEAT 0 + SCX_EV_REENQ_REPEAT 0 SCX_EV_REFILL_SLICE_DFL 456789 SCX_EV_BYPASS_DURATION 0 SCX_EV_BYPASS_DISPATCH 0 @@ -129,9 +129,9 @@ The counters are described in ``kernel/sched/ext/internal.h``; briefly: ``SCX_OPS_ENQ_MIGRATION_DISABLED`` is not set). * ``SCX_EV_REENQ_IMMED``: a task dispatched with ``SCX_ENQ_IMMED`` was re-enqueued because the target CPU was not available for immediate execution. -* ``SCX_EV_REENQ_LOCAL_REPEAT``: a reenqueue of the local DSQ triggered - another reenqueue; recurring counts indicate incorrect ``SCX_ENQ_REENQ`` - handling in the BPF scheduler. +* ``SCX_EV_REENQ_REPEAT``: a reenqueue led to another reenqueue without the + task running in between; recurring counts indicate that the BPF scheduler + keeps re-deciding placements it can't honor. * ``SCX_EV_REFILL_SLICE_DFL``: a task's time slice was refilled with the default value (``SCX_SLICE_DFL``). * ``SCX_EV_BYPASS_DURATION``: total nanoseconds spent in bypass mode. @@ -153,6 +153,7 @@ detailed information: switching_all : 1 switched_all : 1 enable_state : enabled (2) + aborting : False bypass_depth : 0 nr_rejected : 0 enable_seq : 1 diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index b905208942bf..303655d9deaf 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -82,12 +82,38 @@ enum cgroup_lifetime_events { CGROUP_LIFETIME_OFFLINE, }; +/* + * Events on cgroup_task_notifier, data is struct cgroup_task_migrate_ctx. + * MIGRATING fires per task before the migration commits and an error return + * from the chain fails the migration, in which case tasks that were already + * notified receive MIGRATE_CANCELED. MIGRATED fires per task after the + * migration is committed and can't fail. Only migrations that change a task's + * dfl cgroup are reported. + */ +enum cgroup_task_events { + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATED, + CGROUP_TASK_MIGRATE_CANCELED, +}; + +/* + * @src_dcgrp and @dst_dcgrp are @task's dfl cgroups before and after the + * migration. @src_dcgrp is NULL for CGROUP_TASK_MIGRATED as per-task sources + * are not tracked past the commit point. + */ +struct cgroup_task_migrate_ctx { + struct task_struct *task; + struct cgroup *src_dcgrp; + struct cgroup *dst_dcgrp; +}; + extern struct file_system_type cgroup_fs_type; extern struct cgroup_root cgrp_dfl_root; extern struct css_set init_css_set; extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct blocking_notifier_head cgroup_lifetime_notifier; +extern struct blocking_notifier_head cgroup_task_notifier; #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys; #include diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 87e353f7e011..582d7cd4a983 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -58,6 +58,8 @@ enum scx_dsq_id_flags { SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, + SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ + SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; @@ -101,6 +103,7 @@ enum scx_ent_flags { SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */ SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ + SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ /* * Bits 8 to 10 are used to carry task state: @@ -124,7 +127,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 and 13 are used to carry reenqueue reason. In addition to + * Bits 12 to 14 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -132,15 +135,17 @@ enum scx_ent_flags { * KFUNC reenqueued by scx_bpf_dsq_reenq() and friends * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running + * CAP sub-sched cap miss, see p->scx.reenq_reason_* */ SCX_TASK_REENQ_REASON_SHIFT = 12, - SCX_TASK_REENQ_REASON_BITS = 2, + SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, @@ -189,22 +194,26 @@ struct sched_ext_entity { atomic_long_t ops_state; u64 ddsp_dsq_id; u64 ddsp_enq_flags; + u64 ddsp_slice; + u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; u32 dsq_flags; /* protected by DSQ lock */ u32 flags; /* protected by rq lock */ u32 weight; + u32 reenq_cnt; /* reenqueues since last run */ s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; + s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ unsigned long runnable_at; -#ifdef CONFIG_SCHED_CORE - u64 core_sched_at; /* see scx_prio_less() */ +#ifdef CONFIG_EXT_SUB_SCHED + unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ #endif /* @@ -219,10 +228,11 @@ struct sched_ext_entity { /* BPF scheduler modifiable fields */ /* - * Runtime budget in nsecs. This is usually set through - * scx_bpf_dsq_insert() but can also be modified directly by the BPF - * scheduler. Automatically decreased by SCX as the task executes. On - * depletion, a scheduling event is triggered. + * Runtime budget in nsecs - how long the task may hold its cpu. Owned + * by the task's scheduler. Set it when enqueuing via + * scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice(). + * Automatically decreased as the task executes. On depletion a + * scheduling event is triggered. * * This value is cleared to zero if the task is preempted by * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the @@ -239,6 +249,22 @@ struct sched_ext_entity { */ u64 dsq_vtime; + /* + * Out-of-band slice request from scx_bpf_task_set_slice() when the + * caller does not hold the rq lock, applied under the rq lock at the + * next slice consideration. One atomic64 packs the pending flag, the + * issuing sch's id, and the requested slice. See scx_slice_oob_consts. + */ + atomic64_t slice_oob; + + /* + * Sub-sched cap rejected reenq context, valid only while + * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits + * that were needed but missing. @reenq_reason_cid is the target cid. + */ + u64 reenq_reason_caps; + s32 reenq_reason_cid; + /* * If set, reject future sched_setscheduler(2) calls updating the policy * to %SCHED_EXT with -%EACCES. @@ -263,7 +289,7 @@ void sched_ext_dead(struct task_struct *p); void print_scx_info(const char *log_lvl, struct task_struct *p); void scx_softlockup(u32 dur_s); bool scx_hardlockup(int cpu); -bool scx_rcu_cpu_stall(void); +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask); #else /* !CONFIG_SCHED_CLASS_EXT */ @@ -271,12 +297,27 @@ static inline void sched_ext_dead(struct task_struct *p) {} static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {} static inline void scx_softlockup(u32 dur_s) {} static inline bool scx_hardlockup(int cpu) { return false; } -static inline bool scx_rcu_cpu_stall(void) { return false; } +static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; } #endif /* CONFIG_SCHED_CLASS_EXT */ struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + /* + * The sched this tg is on, NULL if none. SCX_TG_INITED tracks whether + * ops.cgroup_init() succeeded on it. When a child sched exits and its + * tgs move to the parent, a failed init leaves the tg on the parent + * with INITED clear (see scx_cgroup_return_subtree()). + * + * This is tracked separately from cgrp->scx_sched because the tg + * hierarchy can diverge from the cgroup2 hierarchy in both lifetime and + * shape. A tg stays online past its cgroup's removal while the + * cgrp->scx_sched rewrites visit only live cgroups, leaving a removed + * cgroup's pointer stale. The cpu controller can also be mounted on + * cgroup1. + */ + struct scx_sched *sched; + u32 flags; /* SCX_TG_* */ u32 weight; u64 bw_period_us; diff --git a/include/trace/events/sched_ext.h b/include/trace/events/sched_ext.h index d1bf5acd59c5..1e54f9564ef3 100644 --- a/include/trace/events/sched_ext.h +++ b/include/trace/events/sched_ext.h @@ -84,6 +84,34 @@ TRACE_EVENT(sched_ext_bypass_lb, ) ); +TRACE_EVENT(sched_ext_exit, + + TP_PROTO(struct scx_sched *sch, __u32 kind), + + TP_ARGS(sch, kind), + + TP_STRUCT__entry( + __string( name, sch->ops.name ) + __field( __s32, level ) + __field( __u64, sub_cgroup_id ) + __string( cgrp_path, sch_cgrp_path(sch) ) + __field( __u32, kind ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->level = sch->level; + __entry->sub_cgroup_id = sch->ops.sub_cgroup_id; + __assign_str(cgrp_path); + __entry->kind = kind; + ), + + TP_printk("sched %s level %d sub_cgroup_id %llu cgrp_path %s kind %u", + __get_str(name), __entry->level, __entry->sub_cgroup_id, + __get_str(cgrp_path), __entry->kind + ) +); + #endif /* _TRACE_SCHED_EXT_H */ /* This part must be outside protection */ diff --git a/init/init_task.c b/init/init_task.c index ba5c2523f7e0..adb207cd987c 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -145,6 +145,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node), .sticky_cpu = -1, .holding_cpu = -1, + .runnable_cpu = -1, .runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node), .runnable_at = INITIAL_JIFFIES, .ddsp_dsq_id = SCX_DSQ_INVALID, diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 35f546a042b1..f294dad43bd7 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -173,6 +173,7 @@ config SCHED_CORE config SCHED_CLASS_EXT bool "Extensible Scheduling Class" depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + select GENERIC_ALLOCATOR select STACKTRACE if STACKTRACE_SUPPORT help This option enables a new scheduler class sched_ext (SCX), which diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 98c536f8b666..c3a12fee7528 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -88,6 +88,8 @@ EXPORT_SYMBOL_GPL(css_set_lock); struct blocking_notifier_head cgroup_lifetime_notifier = BLOCKING_NOTIFIER_INIT(cgroup_lifetime_notifier); +struct blocking_notifier_head cgroup_task_notifier = + BLOCKING_NOTIFIER_INIT(cgroup_task_notifier); DEFINE_SPINLOCK(trace_cgroup_path_lock); char trace_cgroup_path[TRACE_CGROUP_PATH_LEN]; @@ -2676,14 +2678,27 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, return NULL; } +static void cgroup_migrate_notify_canceled(struct css_set *src_cset, + struct task_struct *task) +{ + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = src_cset->dfl_cgrp, + .dst_dcgrp = src_cset->mg_dst_cset->dfl_cgrp, + }; + + blocking_notifier_call_chain(&cgroup_task_notifier, + CGROUP_TASK_MIGRATE_CANCELED, &ctx); +} + /** * cgroup_migrate_execute - migrate a taskset * @mgctx: migration context * - * Migrate tasks in @mgctx as setup by migration preparation functions. - * This function fails iff one of the ->can_attach callbacks fails and - * guarantees that either all or none of the tasks in @mgctx are migrated. - * @mgctx is consumed regardless of success. + * Migrate tasks in @mgctx as setup by migration preparation functions. This + * function fails iff one of the ->can_attach callbacks or CGROUP_TASK_MIGRATING + * notifications fails and guarantees that either all or none of the tasks in + * @mgctx are migrated. @mgctx is consumed regardless of success. */ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) { @@ -2691,6 +2706,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) struct cgroup_subsys *ss; struct task_struct *task, *tmp_task; struct css_set *cset, *tmp_cset; + bool dfl_migration = false; int ssid, failed_ssid, ret; /* check that we can legitimately attach to the cgroup */ @@ -2707,6 +2723,33 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task about the impending migration. An error return fails + * the migration. Only migrations on the default hierarchy are reported: + * a migration modifies either every moved task's dfl cgroup or, on + * cgroup1 or for subtree_control writes, none. + */ + list_for_each_entry(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + dfl_migration = true; + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = cset->dfl_cgrp, + .dst_dcgrp = cset->mg_dst_cset->dfl_cgrp, + }; + + ret = blocking_notifier_call_chain_robust(&cgroup_task_notifier, + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATE_CANCELED, + &ctx); + ret = notifier_to_errno(ret); + if (ret) + goto out_cancel_migrating; + } + } + /* * Now that we're guaranteed success, proceed to move all tasks to * the new cgroup. There are no failure cases after here, so this @@ -2750,9 +2793,41 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task after successful migration. The operation can no + * longer fail and the return value is ignored. The MIGRATING loop + * above explains why only dfl migrations are reported. Per-task + * sources are not tracked past the commit point, so src_dcgrp is + * NULL. + */ + if (dfl_migration) { + list_for_each_entry(cset, &tset->dst_csets, mg_node) { + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .dst_dcgrp = cset->dfl_cgrp, + }; + + blocking_notifier_call_chain( + &cgroup_task_notifier, + CGROUP_TASK_MIGRATED, &ctx); + } + } + } + ret = 0; goto out_release_tset; +out_cancel_migrating: + list_for_each_entry_continue_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + list_for_each_entry_continue_reverse(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + list_for_each_entry_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + } + failed_ssid = CGROUP_SUBSYS_COUNT; out_cancel_attach: if (tset->nr_tasks) { do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { @@ -2976,11 +3051,11 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx) * cgroup_migrate_prepare_dst() on the targets before invoking this * function and following up with cgroup_migrate_finish(). * - * As long as a controller's ->can_attach() doesn't fail, this function is - * guaranteed to succeed. This means that, excluding ->can_attach() - * failure, when migrating multiple targets, the success or failure can be - * decided for all targets by invoking group_migrate_prepare_dst() before - * actually starting migrating. + * As long as a controller's ->can_attach() or a CGROUP_TASK_MIGRATING + * notification doesn't fail, this function is guaranteed to succeed. This + * means that, excluding those failures, when migrating multiple targets, + * the success or failure can be decided for all targets by invoking + * group_migrate_prepare_dst() before actually starting migrating. */ int cgroup_migrate(struct task_struct *leader, bool threadgroup, struct cgroup_mgctx *mgctx) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..415583c35f8c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4900,6 +4900,9 @@ static void __init rcu_dump_rcu_node_tree(void) struct workqueue_struct *rcu_gp_wq; +static struct cpumask rcu_stall_cpumask; +static struct cpumask rcu_exp_stall_cpumask; + void __init rcu_init(void) { int cpu = smp_processor_id(); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..46b6907f1b09 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -578,6 +578,7 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne if (!(READ_ONCE(rnp->expmask) & mask)) continue; ndetected++; + cpumask_set_cpu(cpu, &rcu_exp_stall_cpumask); rdp = per_cpu_ptr(&rcu_data, cpu); pr_cont(" %d-%c%c%c%c", cpu, "O."[!!cpu_online(cpu)], @@ -665,6 +666,8 @@ static void synchronize_rcu_expedited_wait(void) if (rcu_stall_is_suppressed()) continue; + cpumask_clear(&rcu_exp_stall_cpumask); + nbcon_cpu_emergency_enter(); j = jiffies; @@ -675,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_exp_stall_cpumask); } } diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..ebf381936eb1 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -159,7 +159,7 @@ static int __init check_cpu_stall_init(void) early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ -static void panic_on_rcu_stall(void) +static void panic_on_rcu_stall(const struct cpumask *stalled_mask) { static int cpu_stall; @@ -167,7 +167,7 @@ static void panic_on_rcu_stall(void) * Attempt to kick out the BPF scheduler if it's installed and defer * the panic to give the system a chance to recover. */ - if (scx_rcu_cpu_stall()) + if (scx_rcu_cpu_stall(stalled_mask)) return; if (++cpu_stall < sysctl_max_rcu_stall_to_panic) @@ -644,6 +644,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) if (rcu_stall_is_suppressed()) return; + cpumask_clear(&rcu_stall_cpumask); + nbcon_cpu_emergency_enter(); /* @@ -659,6 +661,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) for_each_leaf_node_possible_cpu(rnp, cpu) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { print_cpu_stall_info(cpu); + cpumask_set_cpu(cpu, &rcu_stall_cpumask); ndetected++; } } @@ -700,7 +703,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_stall_cpumask); rcu_force_quiescent_state(); /* Kick them all. */ } @@ -753,7 +756,9 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + cpumask_clear(&rcu_stall_cpumask); + cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask); /* * Attempt to revive the RCU machinery by forcing a context switch. diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index d74b54f81992..2a828725a7f9 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -66,10 +66,13 @@ # include "ext/cid.h" # include "ext/arena.h" # include "ext/idle.h" +# include "ext/sub.h" +# include "ext/inlines.h" # include "ext/ext.c" # include "ext/cid.c" # include "ext/arena.c" # include "ext/idle.c" +# include "ext/sub.c" #endif #include "syscalls.c" diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6544e56925ee..4520d63763d4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -443,6 +443,17 @@ static void __sched_core_flip(bool enabled) sched_core_lock(cpu, &flags); + /* + * A core-wide selection may have the shared rq lock temporarily + * released by a lock-dropping ->pick_task(). Flipping would + * rebind rq_lockp() under it. Wait it out. + */ + while (cpu_rq(cpu)->core->core_pick_in_flight) { + sched_core_unlock(cpu, &flags); + cpu_relax(); + sched_core_lock(cpu, &flags); + } + for_each_cpu(t, smt_mask) cpu_rq(t)->core_enabled = enabled; @@ -6228,7 +6239,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) unsigned long cookie; int i, cpu, occ = 0; struct rq *rq_i; - bool need_sync; + bool need_sync = false; if (!sched_core_enabled(rq)) return __pick_next_task(rq, rf); @@ -6247,6 +6258,8 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) return __pick_next_task(rq, rf); } + rq->core->core_pick_in_flight++; + /* * If there were no {en,de}queues since we picked (IOW, the task * pointers are all still valid), and we haven't scheduled the last @@ -6271,7 +6284,9 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) prev_balance(rq, rf); smt_mask = cpu_smt_mask(cpu); - need_sync = !!rq->core->core_cookie; + +restart: + need_sync |= !!rq->core->core_cookie; /* reset state */ rq->core->core_cookie = 0UL; @@ -6306,10 +6321,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * and there are no cookied tasks running on siblings. */ if (!need_sync) { -restart_single: next = pick_task(rq, rf); - if (unlikely(next == RETRY_TASK)) - goto restart_single; + if (unlikely(next == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } + if (!next->core_cookie) { rq->core_pick = NULL; rq->core_dl_server = NULL; @@ -6329,7 +6349,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * * Tie-break prio towards the current CPU */ -restart_multi: max = NULL; for_each_cpu_wrap(i, smt_mask, cpu) { rq_i = cpu_rq(i); @@ -6343,8 +6362,13 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) update_rq_clock(rq_i); p = pick_task(rq_i, rf); - if (unlikely(p == RETRY_TASK)) - goto restart_multi; + if (unlikely(p == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } rq_i->core_pick = p; rq_i->core_dl_server = rq_i->dl_server; @@ -6450,6 +6474,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) } out_set_next: + rq->core->core_pick_in_flight--; put_prev_set_next_task(rq, rq->donor, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); @@ -6644,6 +6669,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu) core_rq->core_forceidle_seq = rq->core_forceidle_seq; core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; + /* + * A stale leftover would bias the count forever if this CPU later + * returns as its own leader. Move, don't copy. + */ + core_rq->core_pick_in_flight = rq->core_pick_in_flight; + rq->core_pick_in_flight = 0; + /* * Accounting edge for forced idle is handled in pick_next_task(). * Don't need another one here, since the hotplug thread shouldn't @@ -9058,6 +9090,7 @@ void __init sched_init(void) rq->core_forceidle_count = 0; rq->core_forceidle_occupation = 0; rq->core_forceidle_start = 0; + rq->core_pick_in_flight = 0; rq->core_cookie = 0UL; #endif diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c index 5783694ec21d..f7a9f633f435 100644 --- a/kernel/sched/ext/arena.c +++ b/kernel/sched/ext/arena.c @@ -6,8 +6,7 @@ * * Each chunk added to @sch->arena_pool comes from one * bpf_arena_alloc_pages_sleepable() call and is registered at the - * kernel-side mapping address. Callers translate to the BPF-arena form - * themselves if needed. + * kernel-side mapping address. * * Allocations grow the pool on demand. Underlying arena pages are released * when the arena map itself is torn down. @@ -70,8 +69,6 @@ void scx_arena_pool_destroy(struct scx_sched *sch) */ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) { - u64 kern_vm_start; - u32 uaddr32; void *p; int ret; @@ -83,15 +80,8 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) if (!p) return -ENOMEM; - uaddr32 = (u32)(unsigned long)p; - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); -#else - kern_vm_start = 0; -#endif - - ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + ret = gen_pool_add(sch->arena_pool, + (unsigned long)scx_arena_to_kaddr(sch, p), page_cnt * PAGE_SIZE, NUMA_NO_NODE); if (ret) { bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index af83084ec740..39f88deb94bc 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -11,20 +11,26 @@ #include "cid.h" /* - * cid tables. - * - * Pointers are published once on first enable and never revoked. The default - * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits - * before it returns. As long as the BPF scheduler only uses the tables from - * those points onward, it sees a consistent view. + * cid tables. The cid kfuncs are available whether the root scheduler is + * cid-form or cpu-form, the latter to allow gradual migration to cids, so every + * root builds a default mapping. Each root enable allocates a fresh set, builds + * it privately and publishes the __rcu globals below once the layout is final. + * Root disable unpublishes and RCU-frees the set. kfuncs may run before the + * tables are published and must check for NULL. */ -s16 *scx_cid_to_cpu_tbl; -s16 *scx_cpu_to_cid_tbl; -struct scx_cid_topo *scx_cid_topo; +u32 scx_nr_cid_shards; +s16 __rcu *scx_cid_to_cpu_tbl; +s16 __rcu *scx_cpu_to_cid_tbl; +s32 __rcu *scx_cid_to_shard; +s32 __rcu *scx_shard_node; +struct scx_cid_shard __rcu *scx_cid_shard_ranges; +struct scx_cid_topo __rcu *scx_cid_topo; + +static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ - .node_cid = -1, .node_idx = -1, \ + .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ } /* @@ -43,37 +49,128 @@ static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; } -/* Allocate the cid tables once on first enable; never freed. */ -static s32 scx_cid_arrays_alloc(void) +/* + * Compute per-LLC shard layout. Each shard holds at most @shard_size cids, and + * in any case no more than SCX_CID_SHARD_MAX_CPUS. Cores are spread as evenly + * as possible across shards so cpu count is balanced: the first *@nr_large_p + * shards get (*@cores_per_shard_p + 1) cores, the rest get *@cores_per_shard_p. + */ +static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size, + u32 *cores_per_shard_p, u32 *nr_large_p) { - u32 npossible = num_possible_cpus(); - s16 *cid_to_cpu, *cpu_to_cid; - struct scx_cid_topo *cid_topo; + u32 nr_cores = 0, nr_cpus = 0, nr_shards; + int cpu; - if (scx_cid_to_cpu_tbl) - return 0; - - cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); - cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); - cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - - if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { - kfree(cid_to_cpu); - kfree(cpu_to_cid); - kfree(cid_topo); - return -ENOMEM; + for_each_cpu(cpu, llc_cpus) { + nr_cpus++; + if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu) + nr_cores++; } - WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); - WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); - WRITE_ONCE(scx_cid_topo, cid_topo); - return 0; + nr_shards = max_t(u32, 1, DIV_ROUND_UP(nr_cpus, shard_size)); + nr_shards = max_t(u32, nr_shards, + DIV_ROUND_UP(nr_cpus, SCX_CID_SHARD_MAX_CPUS)); + + *cores_per_shard_p = nr_cores / nr_shards; + *nr_large_p = nr_cores % nr_shards; +} + +static void scx_cid_tables_free(struct scx_cid_tables *tbls) +{ + if (!tbls) + return; + kvfree(tbls->cid_to_cpu); + kvfree(tbls->cpu_to_cid); + kvfree(tbls->cid_to_shard); + kvfree(tbls->shard_node); + kvfree(tbls->shard_ranges); + kvfree(tbls->topo); + kfree(tbls); +} + +static void scx_cid_tables_free_rcufn(struct rcu_head *rcu) +{ + scx_cid_tables_free(container_of(rcu, struct scx_cid_tables, rcu)); +} + +static struct scx_cid_tables *scx_cid_alloc_tables(void) +{ + u32 npossible = num_possible_cpus(); + struct scx_cid_tables *tbls; + + tbls = kzalloc_obj(*tbls, GFP_KERNEL); + if (!tbls) + return NULL; + + tbls->cid_to_cpu = kvcalloc(npossible, sizeof(*tbls->cid_to_cpu), GFP_KERNEL); + tbls->cpu_to_cid = kvcalloc(nr_cpu_ids, sizeof(*tbls->cpu_to_cid), GFP_KERNEL); + tbls->cid_to_shard = kvcalloc(npossible, sizeof(*tbls->cid_to_shard), GFP_KERNEL); + tbls->shard_node = kvcalloc(npossible, sizeof(*tbls->shard_node), GFP_KERNEL); + tbls->shard_ranges = kvcalloc(npossible, sizeof(*tbls->shard_ranges), GFP_KERNEL); + tbls->topo = kvcalloc(npossible, sizeof(*tbls->topo), GFP_KERNEL); + + if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard || + !tbls->shard_node || !tbls->shard_ranges || !tbls->topo) { + scx_cid_tables_free(tbls); + return NULL; + } + + return tbls; +} + +/** + * scx_cid_publish_tables - Publish the tables scx_cid_init() built + * + * Called after ops.init_cids() where the layout is final. + */ +void scx_cid_publish_tables(void) +{ + struct scx_cid_tables *tbls = scx_cid_tables; + + lockdep_assert_held(&scx_enable_mutex); + + scx_nr_cid_shards = tbls->nr_shards; + rcu_assign_pointer(scx_cid_to_cpu_tbl, tbls->cid_to_cpu); + rcu_assign_pointer(scx_cpu_to_cid_tbl, tbls->cpu_to_cid); + rcu_assign_pointer(scx_cid_to_shard, tbls->cid_to_shard); + rcu_assign_pointer(scx_shard_node, tbls->shard_node); + rcu_assign_pointer(scx_cid_shard_ranges, tbls->shard_ranges); + rcu_assign_pointer(scx_cid_topo, tbls->topo); +} + +/** + * scx_cid_retire_tables - Unpublish and retire the cid tables + * + * Called by root disable after the readers which dereference without NULL + * checks are drained, inside cpus_read_lock() to exclude the hotplug path. + */ +void scx_cid_retire_tables(void) +{ + struct scx_cid_tables *tbls = scx_cid_tables; + + lockdep_assert_held(&scx_enable_mutex); + lockdep_assert_cpus_held(); + + if (!tbls) + return; + + scx_cid_tables = NULL; + RCU_INIT_POINTER(scx_cid_to_cpu_tbl, NULL); + RCU_INIT_POINTER(scx_cpu_to_cid_tbl, NULL); + RCU_INIT_POINTER(scx_cid_to_shard, NULL); + RCU_INIT_POINTER(scx_shard_node, NULL); + RCU_INIT_POINTER(scx_cid_shard_ranges, NULL); + RCU_INIT_POINTER(scx_cid_topo, NULL); + call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn); } /** * scx_cid_init - build the cid mapping * @sch: the scx_sched being initialized; used as the scx_error() target * + * Build a fresh table set. It becomes visible through scx_cid_publish_tables() + * and is retired by scx_cid_retire_tables() at disable. + * * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by * intersection at each level (parent_scratch & this_level_mask), which keeps * containment correct by construction and naturally splits a physical LLC @@ -88,18 +185,32 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; + struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; - s32 cpu, ret; + s32 next_shard_idx = 0; + u32 shard_size, max_cids; + u32 notopo_in_shard; + s32 notopo_shard_cid, notopo_shard_idx; + s32 cpu, cid, si; /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ BUILD_BUG_ON(NR_CPUS > 8192); lockdep_assert_cpus_held(); + lockdep_assert_held(&scx_enable_mutex); - ret = scx_cid_arrays_alloc(); - if (ret) - return ret; + shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL; + max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS); + + tbls = scx_cid_alloc_tables(); + if (!tbls) + return -ENOMEM; + + scx_cid_tables = tbls; + + for (si = 0; si < num_possible_cpus(); si++) + tbls->shard_node[si] = NUMA_NO_NODE; if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || @@ -111,7 +222,7 @@ s32 scx_cid_init(struct scx_sched *sch) /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ for (cpu = 0; cpu < nr_cpu_ids; cpu++) - scx_cpu_to_cid_tbl[cpu] = -1; + tbls->cpu_to_cid[cpu] = -1; cpumask_copy(to_walk, cpu_online_mask); @@ -142,36 +253,69 @@ s32 scx_cid_init(struct scx_sched *sch) const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); s32 llc_cid = next_cid; s32 llc_idx = next_llc_idx++; + u32 cores_per_shard, nr_large; + u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; + s32 shard_cid, shard_idx; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) return -EINVAL; + calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large); + shard_cid = next_cid; + shard_idx = next_shard_idx++; + tbls->shard_node[shard_idx] = nid; + while (!cpumask_empty(llc_scratch)) { s32 lcpu = cpumask_first(llc_scratch); const struct cpumask *sib = topology_sibling_cpumask(lcpu); s32 core_cid = next_cid; s32 core_idx = next_core_idx++; s32 ccpu; + u32 max_cores, cids_in_core; /* core_scratch = llc_scratch & this core */ cpumask_and(core_scratch, llc_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; + /* + * Advance to a new shard when either core or + * cid count reaches max. The latter bounds + * shard sizes under uneven SMT. Never start an + * empty shard. + */ + cids_in_core = cpumask_weight(core_scratch); + max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0); + if (cores_in_shard && + (cores_in_shard >= max_cores || + cids_in_shard + cids_in_core > max_cids)) { + shard_local++; + cores_in_shard = 0; + cids_in_shard = 0; + shard_cid = next_cid; + shard_idx = next_shard_idx++; + tbls->shard_node[shard_idx] = nid; + } + cores_in_shard++; + cids_in_shard += cids_in_core; + for_each_cpu(ccpu, core_scratch) { s32 cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = ccpu; - scx_cpu_to_cid_tbl[ccpu] = cid; - scx_cid_topo[cid] = (struct scx_cid_topo){ + tbls->cid_to_cpu[cid] = ccpu; + tbls->cpu_to_cid[ccpu] = cid; + tbls->cid_to_shard[cid] = shard_idx; + tbls->topo[cid] = (struct scx_cid_topo){ .core_cid = core_cid, .core_idx = core_idx, .llc_cid = llc_cid, .llc_idx = llc_idx, .node_cid = node_cid, .node_idx = node_idx, + .shard_cid = shard_cid, + .shard_idx = shard_idx, }; cpumask_clear_cpu(ccpu, llc_scratch); @@ -184,21 +328,37 @@ s32 scx_cid_init(struct scx_sched *sch) /* * No-topo section: any possible cpu without a cid - normally just the - * not-online ones. Collect any currently-online cpus that land here in - * @online_no_topo so we can warn about them at the end. + * not-online ones. Pack into shards of up to min(@shard_size, + * SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard + * assignment and the hard cap holds even with a large @shard_size. + * Collect any currently-online cpus that land here in @online_no_topo + * so we can warn about them at the end. */ - for_each_cpu(cpu, cpu_possible_mask) { - s32 cid; + notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS); + notopo_shard_cid = -1; + notopo_shard_idx = -1; - if (__scx_cpu_to_cid(cpu) != -1) + for_each_cpu(cpu, cpu_possible_mask) { + if (tbls->cpu_to_cid[cpu] != -1) continue; if (cpu_online(cpu)) cpumask_set_cpu(cpu, online_no_topo); cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = cpu; - scx_cpu_to_cid_tbl[cpu] = cid; - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + tbls->cid_to_cpu[cid] = cpu; + tbls->cpu_to_cid[cpu] = cid; + + if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) { + notopo_shard_cid = cid; + notopo_shard_idx = next_shard_idx++; + notopo_in_shard = 0; + } + notopo_in_shard++; + + tbls->cid_to_shard[cid] = notopo_shard_idx; + tbls->topo[cid] = SCX_CID_TOPO_NEG; + tbls->topo[cid].shard_cid = notopo_shard_cid; + tbls->topo[cid].shard_idx = notopo_shard_idx; } if (!cpumask_empty(llc_fallback)) @@ -208,6 +368,20 @@ s32 scx_cid_init(struct scx_sched *sch) pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", cpumask_pr_args(online_no_topo)); + /* + * Fill cid_shard_ranges[] from cid_to_shard[]. Shards are contiguous + * cid ranges by construction: base_cid is the first cid landing in a + * shard, nr_cids is the count. + */ + for (cid = 0; cid < next_cid; cid++) { + s32 sidx = tbls->cid_to_shard[cid]; + + if (tbls->shard_ranges[sidx].nr_cids == 0) + tbls->shard_ranges[sidx].base_cid = cid; + tbls->shard_ranges[sidx].nr_cids++; + } + + tbls->nr_shards = next_shard_idx; return 0; } @@ -253,50 +427,76 @@ void scx_cmask_fill(struct scx_cmask *m) m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; } -/** - * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask - * @src: source cpumask - * @dst: cmask to write - * - * Clear @dst's active range and set the bit for each cid whose cpu is in - * @src and lies within that range. Out-of-range cids are silently ignored. +/* + * Return the index of the largest entry in @counts, or NUMA_NO_NODE if all + * entries are zero. Ties resolve to the lowest index. */ -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) +static s32 pick_max_node(const u32 *counts, u32 n) { - s32 cpu; + s32 best = NUMA_NO_NODE; + u32 best_count = 0, i; - scx_cmask_clear(dst); - for_each_cpu(cpu, src) { - s32 cid = __scx_cpu_to_cid(cpu); - - if (cid >= 0) - __scx_cmask_set(cid, dst); + for (i = 0; i < n; i++) { + if (counts[i] > best_count) { + best_count = counts[i]; + best = i; + } } + return best; } __bpf_kfunc_start_defs(); /** - * scx_bpf_cid_override - Install an explicit cpu->cid mapping - * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) - * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes + * scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info + * @cpu_to_cid__arena: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid_cnt: number of entries, must be nr_cpu_ids + * @shard_start__arena: array of first-cid-of-each-shard, one entry per shard + * @shard_start_cnt: number of shards * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * May only be called from ops.init() of the root scheduler. Replace the - * topology-probed cid mapping with the caller-provided one. Each possible cpu - * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. - * On invalid input, trigger scx_error() to abort the scheduler. + * May only be called from ops.init_cids() of the root scheduler. Replace the + * topology-probed cid mapping and shard layout with caller-provided ones. Each + * possible cpu must map to a unique cid in [0, num_possible_cpus()). The shard + * starts must be strictly increasing with the first entry 0 and all values < + * num_possible_cpus(). The last shard extends to num_possible_cpus() and no + * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info + * (core/LLC/node) is cleared and the shard layout is set from the input. On + * invalid input, abort the scheduler. */ -__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, + const s32 *shard_start__arena, u32 shard_start_cnt, const struct bpf_prog_aux *aux) { cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; + u32 *node_counts __free(kfree) = NULL; + s32 *cpu_to_cid __free(kfree) = NULL; + s32 *shard_start __free(kfree) = NULL; + u32 npossible = num_possible_cpus(); + struct scx_cid_tables *tbls; struct scx_sched *sch; + u32 nr_shards = shard_start_cnt; bool alloced; - s32 cpu, cid; + s32 cpu, cid, si; - /* GFP_KERNEL alloc must happen before the rcu read section */ + /* + * GFP_KERNEL allocs must happen before the rcu read section. Snapshot + * the BPF-supplied arrays so a concurrent arena write can't change + * them between validation and use. + * + * The BPF-supplied counts size the snapshots and thus the arena reads. + * Gate the copies on the count bounds, reported below once @sch is + * available. The bounded reads, at most 32KB, stay within the guard + * region that arena fault recovery covers. + */ alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); + node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL); + if (cpu_to_cid_cnt == nr_cpu_ids) + cpu_to_cid = kmemdup(cpu_to_cid__arena, cpu_to_cid_cnt * sizeof(s32), + GFP_KERNEL); + if (nr_shards && nr_shards <= npossible) + shard_start = kmemdup(shard_start__arena, nr_shards * sizeof(s32), + GFP_KERNEL); guard(rcu)(); @@ -304,22 +504,59 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, if (unlikely(!sch)) return; - if (!alloced) { - scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); + /* called from ops.init_cids(), so the tables exist and are unpublished */ + lockdep_assert_held(&scx_enable_mutex); + tbls = scx_cid_tables; + + if (cpu_to_cid_cnt != nr_cpu_ids) { + scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %u entries, got %u", + nr_cpu_ids, cpu_to_cid_cnt); return; } - if (scx_parent(sch)) { - scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); + if (!nr_shards || nr_shards > npossible) { + scx_error(sch, "scx_bpf_cid_override: invalid shard_start count %u", + nr_shards); return; } - if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", - nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); + if (!alloced || !node_counts || !cpu_to_cid || !shard_start) { + scx_error(sch, "scx_bpf_cid_override: allocation failed"); return; } + /* validate shard_start[]: starts at 0, strictly increasing, in range */ + if (shard_start[0] != 0) { + scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d", + shard_start[0]); + return; + } + for (si = 1; si < nr_shards; si++) { + if (shard_start[si] <= shard_start[si - 1]) { + scx_error(sch, "scx_bpf_cid_override: shard_start not increasing at [%d]", + si); + return; + } + if (shard_start[si] >= npossible) { + scx_error(sch, "scx_bpf_cid_override: shard_start[%d]=%d >= %u", + si, shard_start[si], npossible); + return; + } + if (shard_start[si] - shard_start[si - 1] > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d", + si - 1, shard_start[si] - shard_start[si - 1], + SCX_CID_SHARD_MAX_CPUS); + return; + } + } + if (npossible - shard_start[nr_shards - 1] > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d", + nr_shards - 1, npossible - shard_start[nr_shards - 1], + SCX_CID_SHARD_MAX_CPUS); + return; + } + + /* validate first so that invalid input leaves the tables untouched */ for_each_possible_cpu(cpu) { s32 c = cpu_to_cid[cpu]; @@ -329,13 +566,56 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, scx_error(sch, "cid %d assigned to multiple cpus", c); return; } - scx_cpu_to_cid_tbl[cpu] = c; - scx_cid_to_cpu_tbl[c] = cpu; } - /* Invalidate stale topo info - the override carries no topology. */ - for (cid = 0; cid < num_possible_cpus(); cid++) - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + for_each_possible_cpu(cpu) { + s32 c = cpu_to_cid[cpu]; + + tbls->cpu_to_cid[cpu] = c; + tbls->cid_to_cpu[c] = cpu; + } + + /* + * Derive shard_node[] by majority count: an overridden shard may + * span NUMA nodes, so assign each to the node that owns the most cpus. + */ + for (si = 0; si < nr_shards; si++) { + u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible; + + memset(node_counts, 0, nr_node_ids * sizeof(*node_counts)); + for (cid = shard_start[si]; cid < end; cid++) { + s32 node = cpu_to_node(tbls->cid_to_cpu[cid]); + + if (numa_valid_node(node)) + node_counts[node]++; + } + tbls->shard_node[si] = pick_max_node(node_counts, nr_node_ids); + } + + /* + * Invalidate stale topo info and install shard layout from + * @shard_start. Walk shards to derive shard_cid/shard_idx for each cid. + */ + si = 0; + for (cid = 0; cid < npossible; cid++) { + if (si + 1 < nr_shards && cid >= shard_start[si + 1]) + si++; + tbls->cid_to_shard[cid] = si; + tbls->topo[cid] = SCX_CID_TOPO_NEG; + tbls->topo[cid].shard_cid = shard_start[si]; + tbls->topo[cid].shard_idx = si; + } + + /* Rebuild shard_ranges[] for the new layout. */ + memset(tbls->shard_ranges, 0, npossible * sizeof(*tbls->shard_ranges)); + for (si = 0; si < nr_shards; si++) { + u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible; + + tbls->shard_ranges[si].base_cid = shard_start[si]; + tbls->shard_ranges[si].nr_cids = end - shard_start[si]; + } + + tbls->nr_shards = nr_shards; } /** @@ -395,21 +675,25 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero * @dst bits that lie outside @src's range. * - * The _RACY variants are otherwise identical to their non-racy counterpart but - * read @src word-by-word via data_race(). Memory ordering with concurrent - * writers is the caller's responsibility. + * Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src + * locklessly. Memory ordering against concurrent writers is the caller's + * responsibility. */ enum cmask_op2 { /* mutating */ CMASK_OP2_AND, CMASK_OP2_OR, - CMASK_OP2_OR_RACY, CMASK_OP2_COPY, - CMASK_OP2_COPY_RACY, CMASK_OP2_ANDNOT, /* predicates - short-circuit when the per-word result is true */ CMASK_OP2_SUBSET, CMASK_OP2_INTERSECTS, + /* + * @a is a BPF-arena cmask. Words on @a use READ_ONCE/WRITE_ONCE since + * BPF may read/write concurrently. See scx_cmask_ref_or() / _copy(). + */ + CMASK_OP2_REF_OR, + CMASK_OP2_REF_COPY, }; static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) @@ -422,28 +706,28 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, { switch (op) { case CMASK_OP2_AND: - *av &= ~mask | *bp; + WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp))); return false; case CMASK_OP2_OR: - *av |= *bp & mask; - return false; - case CMASK_OP2_OR_RACY: - *av |= data_race(*bp) & mask; + WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_COPY: - *av = (*av & ~mask) | (*bp & mask); - return false; - case CMASK_OP2_COPY_RACY: - *av = (*av & ~mask) | (data_race(*bp) & mask); + WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_ANDNOT: - *av &= ~(*bp & mask); + WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_SUBSET: /* stop on the first bit in @sub not set in @super */ - return (*bp & ~*av) & mask; + return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask; case CMASK_OP2_INTERSECTS: - return (*av & *bp) & mask; + return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask; + case CMASK_OP2_REF_OR: + WRITE_ONCE(*av, READ_ONCE(*av) | (READ_ONCE(*bp) & mask)); + return false; + case CMASK_OP2_REF_COPY: + WRITE_ONCE(*av, (READ_ONCE(*av) & ~mask) | (READ_ONCE(*bp) & mask)); + return false; } unreachable(); } @@ -504,7 +788,7 @@ static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, { switch (op) { case CMASK_OP1_ANY_SET: - return *ap & mask; + return READ_ONCE(*ap) & mask; } unreachable(); } @@ -556,39 +840,12 @@ void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) src->bits, src->base, src->nr_cids, CMASK_OP2_OR); } -/** - * scx_cmask_or_racy - OR @src into @dst, reading @src without locking - * - * @src is read word-by-word through data_race(). Same per-bit independence - * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the - * caller's responsibility. - */ -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); -} - void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) { cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); } -/** - * scx_cmask_copy_racy - Snapshot @src into @dst without locking - * - * @src is read word-by-word through data_race(). Head/tail masking matches - * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates - * just leave some bits fresher than others. Memory ordering with writers is - * the caller's responsibility. - */ -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); -} - void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) { cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, @@ -660,33 +917,36 @@ bool scx_cmask_empty(const struct scx_cmask *m) * * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. + * are set to -1. All fields are also set to -1 when no cid tables have been + * published yet, which a program may observe while racing the root enable. */ __bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, const struct bpf_prog_aux *aux) { + struct scx_cid_topo *topo; struct scx_sched *sch; guard(rcu)(); sch = scx_prog_sched(aux); - if (unlikely(!sch) || !cid_valid(sch, cid)) { + topo = rcu_dereference(scx_cid_topo); + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { *out__uninit = SCX_CID_TOPO_NEG; return; } - *out__uninit = READ_ONCE(scx_cid_topo)[cid]; + *out__uninit = topo[cid]; } __bpf_kfunc_end_defs(); -BTF_KFUNCS_START(scx_kfunc_ids_init) +BTF_KFUNCS_START(scx_kfunc_ids_init_cids) BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -BTF_KFUNCS_END(scx_kfunc_ids_init) +BTF_KFUNCS_END(scx_kfunc_ids_init_cids) -static const struct btf_kfunc_id_set scx_kfunc_set_init = { +static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { .owner = THIS_MODULE, - .set = &scx_kfunc_ids_init, + .set = &scx_kfunc_ids_init_cids, .filter = scx_kfunc_context_filter, }; @@ -701,9 +961,205 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { .set = &scx_kfunc_ids_cid, }; +/** + * scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask + * @sch: scheduler whose arena hosts @src + * @src: BPF-supplied cmask, rebased to its kernel address + * @ref: output ref + * + * Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary + * because BPF may mutate the live header asynchronously. + * + * Return 0 on success, -EINVAL if the range is out of bounds or @alloc_words + * doesn't cover it. + */ +int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, + struct scx_cmask_ref *ref) +{ + u32 base, nr_cids, alloc_words, npossible = num_possible_cpus(); + s32 *cid_to_shard; + + base = READ_ONCE(src->base); + nr_cids = READ_ONCE(src->nr_cids); + alloc_words = READ_ONCE(src->alloc_words); + + if (unlikely(base >= npossible || nr_cids > npossible - base || + SCX_CMASK_NR_WORDS(nr_cids) > alloc_words)) + return -EINVAL; + + ref->sch = sch; + ref->src = (struct scx_cmask *)src; + ref->base = base; + ref->nr_cids = nr_cids; + + cid_to_shard = rcu_dereference_all(scx_cid_to_shard); + ref->shard_first = cid_to_shard[base]; + if (likely(nr_cids)) + ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1; + else + ref->shard_end = ref->shard_first; + + return 0; +} + +/** + * scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask + * @sch: scheduler the cmask belongs to + * @m: kernel address of the target cmask, storage sized for @nr_cids at @base + * @base: first cid of the active range + * @nr_cids: active range length + * @ref: output ref + * + * Like scx_cmask_ref_init() but the geometry is supplied by the caller, not + * read from @m's header, so a concurrent BPF write to the header can't steer + * later sizing or offsets. Rewrite the header from the trusted geometry and + * bind @ref to it. + */ +void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, + u32 base, u32 nr_cids, struct scx_cmask_ref *ref) +{ + s32 *cid_to_shard; + + WRITE_ONCE(m->base, base); + WRITE_ONCE(m->nr_cids, nr_cids); + WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids)); + + ref->sch = sch; + ref->src = m; + ref->base = base; + ref->nr_cids = nr_cids; + + cid_to_shard = rcu_dereference_all(scx_cid_to_shard); + ref->shard_first = cid_to_shard[base]; + if (likely(nr_cids)) + ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1; + else + ref->shard_end = ref->shard_first; +} + +/** + * scx_cmask_ref_shard - Read one shard from @ref into @out + * @ref: validated ref + * @shard_idx: target shard, in [@ref->shard_first, @ref->shard_end) + * @out: output cmask whose @out->alloc_words must hold the shard + * + * Set @out to the intersection of @ref's range with @shard_idx's cid range, + * with bits[] read from @ref->src via READ_ONCE. Empty intersection sets + * @out->nr_cids to 0. scx_error()s on @ref's sched if @out can't hold the + * shard. + */ +void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, + struct scx_cmask *out) +{ + const struct scx_cid_shard *shard = + &rcu_dereference_all(scx_cid_shard_ranges)[shard_idx]; + u32 shard_base = shard->base_cid; + u32 shard_end = shard_base + shard->nr_cids; + u32 isect_base, isect_end, nr_words, src_off, wi; + u64 head_mask, tail_mask; + + isect_base = max(ref->base, shard_base); + isect_end = min(ref->base + ref->nr_cids, shard_end); + + if (isect_base >= isect_end) { + out->base = shard_base; + out->nr_cids = 0; + return; + } + + nr_words = ((isect_end - 1) / 64) - (isect_base / 64) + 1; + if (nr_words > out->alloc_words) { + scx_error(ref->sch, "scx_cmask_ref_shard: out alloc_words=%u < %u for shard %d", + out->alloc_words, nr_words, shard_idx); + out->base = shard_base; + out->nr_cids = 0; + return; + } + + out->base = isect_base; + out->nr_cids = isect_end - isect_base; + src_off = (isect_base / 64) - (ref->base / 64); + + for (wi = 0; wi < nr_words; wi++) + out->bits[wi] = READ_ONCE(ref->src->bits[src_off + wi]); + + head_mask = GENMASK_U64(63, isect_base & 63); + out->bits[0] &= head_mask; + tail_mask = GENMASK_U64((isect_end - 1) & 63, 0); + out->bits[nr_words - 1] &= tail_mask; +} + +/** + * scx_cmask_ref_or - OR @src into the arena cmask referenced by @ref + * @ref: validated ref + * @src: stable kernel cmask + * + * Bits inside the intersection of @ref's snapshotted range with @src's range + * are OR'd into @ref->src and bits outside are left unchanged. Stores on + * @ref->src use WRITE_ONCE since BPF may read/write concurrently. + */ +void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src) +{ + cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_REF_OR); +} + +/** + * scx_cmask_ref_copy - Copy @src into the arena cmask referenced by @ref + * @ref: validated ref + * @src: stable kernel cmask + * + * Bits inside the intersection of @ref's snapshotted range with @src's range + * take @src's values and bits outside are left unchanged. Stores on @ref->src + * use WRITE_ONCE since BPF may read/write concurrently. + */ +void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src) +{ + cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY); +} + +/** + * scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask + * @ref: kern-bound ref, see scx_cmask_ref_init_kern() + * @cpumask: cpus to translate into cids + * + * Write @ref's active range one word at a time, setting each cid's bit when + * its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry + * and stores use WRITE_ONCE since BPF may read concurrently, so the arena + * header is never read. + */ +void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref, + const struct cpumask *cpumask) +{ + struct scx_cmask *m = ref->src; + u32 base = ref->base, nr_cids = ref->nr_cids; + u32 wi, nr_words; + + if (!nr_cids) + return; + + nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1; + for (wi = 0; wi < nr_words; wi++) { + u32 word_first_cid = (base / 64 + wi) * 64; + u64 word = 0; + u32 bit; + + for (bit = 0; bit < 64; bit++) { + u32 cid = word_first_cid + bit; + + if (cid < base || cid >= base + nr_cids) + continue; + if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask)) + word |= BIT_U64(bit); + } + WRITE_ONCE(m->bits[wi], word); + } +} + int scx_cid_kfunc_init(void) { - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 9c4f4b907f12..2fe2311a0f99 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -48,25 +48,40 @@ struct scx_sched; * See the comment above the table definitions in cid.c for the * memory-ordering and visibility contract. */ -extern s16 *scx_cid_to_cpu_tbl; -extern s16 *scx_cpu_to_cid_tbl; -extern struct scx_cid_topo *scx_cid_topo; -extern struct btf_id_set8 scx_kfunc_ids_init; +struct scx_cid_tables { + u32 nr_shards; + s16 *cid_to_cpu; /* [num_possible_cpus()] */ + s16 *cpu_to_cid; /* [nr_cpu_ids] */ + s32 *cid_to_shard; /* [num_possible_cpus()] */ + s32 *shard_node; /* [num_possible_cpus()] */ + struct scx_cid_shard *shard_ranges; /* [num_possible_cpus()] */ + struct scx_cid_topo *topo; /* [num_possible_cpus()] */ + struct rcu_head rcu; +}; + +extern u32 scx_nr_cid_shards; +extern s16 __rcu *scx_cid_to_cpu_tbl; +extern s16 __rcu *scx_cpu_to_cid_tbl; +extern s32 __rcu *scx_cid_to_shard; +extern s32 __rcu *scx_shard_node; +extern struct scx_cid_shard __rcu *scx_cid_shard_ranges; +extern struct scx_cid_topo __rcu *scx_cid_topo; +extern struct btf_id_set8 scx_kfunc_ids_init_cids; +extern struct btf_id_set8 scx_kfunc_ids_cid; void scx_cmask_clear(struct scx_cmask *m); void scx_cmask_fill(struct scx_cmask *m); void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); bool scx_cmask_empty(const struct scx_cmask *m); s32 scx_cid_init(struct scx_sched *sch); +void scx_cid_publish_tables(void); +void scx_cid_retire_tables(void); int scx_cid_kfunc_init(void); -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); /** * cid_valid - Verify a cid value, to be used on ops input args @@ -88,14 +103,12 @@ static inline bool cid_valid(struct scx_sched *sch, s32 cid) * __scx_cid_to_cpu - Unchecked cid->cpu table lookup * @cid: cid to look up. Must be in [0, num_possible_cpus()). * - * Intended for callsites that have already validated @cid and that hold a - * non-NULL @sch from scx_prog_sched() - a live sched implies the table has - * been allocated, so no NULL check is needed here. + * Intended for callsites that have already validated @cid and that run on a + * live scheduler, which guarantees the tables are published and stable. */ static inline s32 __scx_cid_to_cpu(s32 cid) { - /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ - return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; + return rcu_dereference_all(scx_cid_to_cpu_tbl)[cid]; } /** @@ -106,7 +119,7 @@ static inline s32 __scx_cid_to_cpu(s32 cid) */ static inline s32 __scx_cpu_to_cid(s32 cpu) { - return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; + return rcu_dereference_all(scx_cpu_to_cid_tbl)[cpu]; } /** @@ -115,15 +128,19 @@ static inline s32 __scx_cpu_to_cid(s32 cpu) * @cid: cid to look up * * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers - * scx_error() on @sch. The cid arrays are allocated on first scheduler enable - * and never freed, so the returned cpu is stable for the lifetime of the loaded - * scheduler. + * scx_error() on @sch. The mapping is stable while the scheduler is live. + * + * Return -EINVAL without triggering scx_error() if no tables have been + * published yet, which a prog-facing kfunc can observe while racing the root + * scheduler enable. */ static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) { - if (!cid_valid(sch, cid)) + s16 *tbl = rcu_dereference_all(scx_cid_to_cpu_tbl); + + if (!cid_valid(sch, cid) || unlikely(!tbl)) return -EINVAL; - return __scx_cid_to_cpu(cid); + return tbl[cid]; } /** @@ -132,13 +149,15 @@ static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) * @cpu: cpu to look up * * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers - * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). + * scx_error() on @sch. Same usage rules as scx_cid_to_cpu(). */ static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) { - if (!scx_cpu_valid(sch, cpu, NULL)) + s16 *tbl = rcu_dereference_all(scx_cpu_to_cid_tbl); + + if (!scx_cpu_valid(sch, cpu, NULL) || unlikely(!tbl)) return -EINVAL; - return __scx_cpu_to_cid(cpu); + return tbl[cpu]; } /** @@ -291,4 +310,15 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) return scx_cid_to_cpu(sch, cpu_or_cid); } +int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, + struct scx_cmask_ref *ref); +void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, + u32 base, u32 nr_cids, struct scx_cmask_ref *ref); +void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, + struct scx_cmask *out); +void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref, + const struct cpumask *cpumask); +void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src); +void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src); + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 18183062f751..10af28a9f2c0 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -19,8 +19,10 @@ #include "cid.h" #include "arena.h" #include "idle.h" +#include "sub.h" +#include "inlines.h" -static DEFINE_RAW_SPINLOCK(scx_sched_lock); +DEFINE_RAW_SPINLOCK(scx_sched_lock); /* * NOTE: sched_ext is in the process of growing multiple scheduler support and @@ -36,17 +38,17 @@ struct scx_sched __rcu *scx_root; * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. * Readers can hold either or rcu_read_lock(). */ -static LIST_HEAD(scx_sched_all); +LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { +const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; -static struct rhashtable scx_sched_hash; +struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ @@ -68,9 +70,9 @@ static DEFINE_RAW_SPINLOCK(scx_tasks_lock); static LIST_HEAD(scx_tasks); /* ops enable/disable */ -static DEFINE_MUTEX(scx_enable_mutex); +DEFINE_MUTEX(scx_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +DEFINE_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); static DEFINE_RAW_SPINLOCK(scx_bypass_lock); static bool scx_init_task_enabled; @@ -78,6 +80,14 @@ static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); +/* + * Gates cgroup ops delivery. Set at the end of the cgroup init phase of root + * enable and cleared before root disable starts tearing down tasks, both under + * scx_cgroup_lock(). Holding cgroup_lock() and seeing %true guarantees no race + * against root tearing down tasks. + */ +bool scx_cgroup_enabled; + /* * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler * and the tid->task table is live. Wraps the static key so callers don't @@ -95,13 +105,29 @@ static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); /* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); +/* is @dsq synchronized by the containing rq lock instead of dsq->lock? */ +static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) +{ + switch (dsq->id) { + case SCX_DSQ_LOCAL: + case SCX_DSQ_REJECT: + case SCX_DSQ_RESCUE: + return true; + default: + return false; + } +} + +/* Cursor for unique scx_sched instance ids. id 0 is reserved. */ +static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0); + #ifdef CONFIG_EXT_SUB_SCHED /* * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit * tasks for the sub-sched being enabled. Use a global variable instead of a * per-task field as all enables are serialized. */ -static struct scx_sched *scx_enabling_sub_sched; +struct scx_sched *scx_enabling_sub_sched; #else #define scx_enabling_sub_sched (struct scx_sched *)NULL #endif /* CONFIG_EXT_SUB_SCHED */ @@ -171,15 +197,6 @@ static const struct rhashtable_params dsq_hash_params = { static LLIST_HEAD(dsqs_to_free); -/* string formatting from BPF */ -struct scx_bstr_buf { - u64 data[MAX_BPRINTF_VARARGS]; - char line[SCX_EXIT_MSG_LEN]; -}; - -static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); -static struct scx_bstr_buf scx_exit_bstr_buf; - /* ops debug dump */ static DEFINE_RAW_SPINLOCK(scx_dump_lock); @@ -242,7 +259,6 @@ MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microsecond static void run_deferred(struct rq *rq); static bool task_dead_and_done(struct task_struct *p); -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); __printf(5, 6) bool __scx_exit(struct scx_sched *sch, @@ -272,58 +288,6 @@ static bool u32_before(u32 a, u32 b) return (s32)(a - b) < 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: sched whose descendants to walk - * - * To be used by scx_for_each_descendant_pre(). Find the next descendant to - * visit for pre-order traversal of @root's descendants. @root is included in - * the iteration and the first node to be visited. - */ -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, - struct scx_sched *root) -{ - struct scx_sched *next; - - lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); - pos = scx_parent(pos); - } - - return NULL; -} - -static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) -{ - return rhashtable_lookup(&scx_sched_hash, &cgroup_id, - scx_sched_hash_params); -} - -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) -{ - rcu_assign_pointer(p->scx.sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /** * scx_is_descendant - Test whether sched is a descendant * @sch: sched to test @@ -331,26 +295,13 @@ static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *s * * Test whether @sch is a descendant of @ancestor. */ -static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) +bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) { if (sch->level < ancestor->level) return false; return sch->ancestors[ancestor->level] == ancestor; } -/** - * scx_for_each_descendant_pre - pre-order walk of a sched's descendants - * @pos: iteration cursor - * @root: sched to walk the descendants of - * - * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. - */ -#define scx_for_each_descendant_pre(pos, root) \ - for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ - (pos) = scx_next_descendant_pre((pos), (root))) - static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) { return &sch->pnode[cpu_to_node(cpu)]->global_dsq; @@ -369,11 +320,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) -{ - return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; -} - static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { #ifdef CONFIG_EXT_SUB_SCHED @@ -392,27 +338,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c sch = scx_parent(sch); #endif /* CONFIG_EXT_SUB_SCHED */ - return bypass_dsq(sch, cpu); -} - -/** - * bypass_dsp_enabled - Check if bypass dispatch path is enabled - * @sch: scheduler to check - * - * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled - * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors - * are bypassing. In the former case, the ancestor is not itself bypassing but - * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, - * the ancestor's bypass dispatch path must be active even though its own - * bypass_depth remains zero. - * - * This function checks bypass_dsp_enable_depth which is managed separately from - * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * disable_bypass_dsp(). - */ -static bool bypass_dsp_enabled(struct scx_sched *sch) -{ - return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); + return scx_bypass_dsq(sch, cpu); } /** @@ -451,9 +377,9 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) * If we're in the dispatch path holding rq lock, $curr may or may not * be ready depending on whether the on-going dispatch decides to extend * $curr's slice. We say yes here and resolve it at the end of dispatch. - * See balance_one(). + * See dispatch_one(). */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* @@ -461,8 +387,16 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) - return true; + if (enq_flags & SCX_ENQ_PREEMPT) { + struct task_struct *curr = rq->curr; + + /* + * A protected slice refuses the preemption and the cpu stays + * occupied. See rq_owned_post_enq(). + */ + return curr->sched_class != &ext_sched_class || + likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + } /* * @rq is either in transition to or running an SCX task and can't go @@ -479,12 +413,27 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); +/* + * Under core scheduling, a pick that releases the rq lock invalidates the + * core-wide selection it is part of. Count the releases so that the core-sched + * pick can tell whether one happened across dispatch. + */ +static void scx_rq_lock_drop(struct rq *rq) +{ + lockdep_assert_rq_held(rq); +#ifdef CONFIG_SCHED_CORE + if (sched_core_enabled(rq)) + rq->scx.lock_drop_seq++; +#endif +} + static void switch_rq_lock(struct rq *from, struct rq *to) { bool tracked = scx_locked_rq() == from; if (tracked) update_locked_rq(NULL); + scx_rq_lock_drop(from); raw_spin_rq_unlock(from); raw_spin_rq_lock(to); if (tracked) @@ -512,27 +461,23 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - WARN_ON_ONCE(current->scx.kf_tasks[0]); - current->scx.kf_tasks[0] = task; - if (rq) - update_locked_rq(rq); - if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - /* - * Build the per-CPU arena cmask and hand BPF its arena address. - * Caller holds the rq lock with IRQs disabled, which makes us - * the sole user of the scratch area. - */ - scx_cpumask_to_cmask(cpumask, kern_va); - sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); - } else { - sch->ops.set_cpumask(task, cpumask); - } + struct scx_cmask_ref ref; - if (rq) - update_locked_rq(NULL); - current->scx.kf_tasks[0] = NULL; + /* + * Build the per-cpu arena cmask from kernel geometry via @ref, + * never reading its BPF-writable header. set_cmask()'s __arena + * argument takes the kernel address and the struct_ops + * trampoline rebases it into BPF's arena pointer form. The rq + * lock makes this cpu the sole kernel writer. + */ + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); + } else { + SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); + } } enum scx_dsq_iter_flags { @@ -688,12 +633,12 @@ struct bpf_iter_scx_dsq { } __attribute__((aligned(8))); -static u32 scx_get_task_state(const struct task_struct *p) +u32 scx_get_task_state(const struct task_struct *p) { return p->scx.flags & SCX_TASK_STATE_MASK; } -static void scx_set_task_state(struct task_struct *p, u32 state) +void scx_set_task_state(struct task_struct *p, u32 state) { u32 prev_state = scx_get_task_state(p); bool warn = false; @@ -733,23 +678,6 @@ static void scx_set_task_state(struct task_struct *p, u32 state) p->scx.flags |= state; } -/* - * SCX task iterator. - */ -struct scx_task_iter { - struct sched_ext_entity cursor; - struct task_struct *locked_task; - struct rq *rq; - struct rq_flags rf; - u32 cnt; - bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED - struct cgroup *cgrp; - struct cgroup_subsys_state *css_pos; - struct css_task_iter css_iter; -#endif -}; - /** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init @@ -778,7 +706,7 @@ struct scx_task_iter { * All tasks which existed when the iteration started are guaranteed to be * visited as long as they are not dead. */ -static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) { memset(iter, 0, sizeof(*iter)); @@ -817,7 +745,7 @@ static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) * This function can be safely called anytime during an iteration. The next * iterator operation will automatically restore the necessary locking. */ -static void scx_task_iter_unlock(struct scx_task_iter *iter) +void scx_task_iter_unlock(struct scx_task_iter *iter) { __scx_task_iter_rq_unlock(iter); if (iter->list_locked) { @@ -860,7 +788,7 @@ static void scx_task_iter_relock(struct scx_task_iter *iter, * which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */ -static void scx_task_iter_stop(struct scx_task_iter *iter) +void scx_task_iter_stop(struct scx_task_iter *iter) { #ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) { @@ -935,7 +863,7 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) * whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */ -static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) { struct task_struct *p; @@ -990,42 +918,6 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) return NULL; } -/** - * scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This can be used when preemption is not disabled. - */ -#define scx_add_event(sch, name, cnt) do { \ - this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, (cnt)); \ -} while(0) - -/** - * __scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This should be used only when preemption is disabled. - */ -#define __scx_add_event(sch, name, cnt) do { \ - __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, cnt); \ -} while(0) - -/** - * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' - * @dst_e: destination event stats - * @src_e: source event stats - * @kind: a kind of event to be aggregated - */ -#define scx_agg_event(dst_e, src_e, kind) do { \ - (dst_e)->kind += READ_ONCE((src_e)->kind); \ -} while(0) - /** * scx_dump_event - Dump an event 'kind' in 'events' to 's' * @s: output seq_buf @@ -1033,7 +925,7 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) * @kind: a kind of event to dump */ #define scx_dump_event(s, events, kind) do { \ - dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ + scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ } while (0) @@ -1100,28 +992,6 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) } } -/** - * ops_sanitize_err - Sanitize a -errno value - * @sch: scx_sched to error out on error - * @ops_name: operation to blame on failure - * @err: -errno value to sanitize - * - * Verify @err is a valid -errno. If not, trigger scx_error() and return - * -%EPROTO. This is necessary because returning a rogue -errno up the chain can - * cause misbehaviors. For an example, a large negative return from - * ops.init_task() triggers an oops when passed up the call chain because the - * value fails IS_ERR() test after being encoded with ERR_PTR() and then is - * handled as a pointer. - */ -static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) -{ - if (err < 0 && err >= -MAX_ERRNO) - return err; - - scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); - return -EPROTO; -} - static void deferred_bal_cb_workfn(struct rq *rq) { run_deferred(rq); @@ -1133,6 +1003,7 @@ static void deferred_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(rq); run_deferred(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } @@ -1149,7 +1020,7 @@ static void schedule_deferred(struct rq *rq) /* * This is the fallback when schedule_deferred_locked() can't use * the cheaper balance callback or wakeup hook paths (the target - * CPU is not in balance or wakeup). Currently, this is primarily + * CPU is not in dispatch or wakeup). Currently, this is primarily * hit by reenqueue operations targeting a remote CPU. * * Queue on the target CPU. The deferred work can run from any CPU @@ -1185,31 +1056,31 @@ static void schedule_deferred_locked(struct rq *rq) return; /* - * If in balance, the balance callbacks will be called before rq lock is - * released. Schedule one. + * If in dispatch, the balance callbacks will be called before rq lock + * is released. Schedule one. * * * We can't directly insert the callback into the * rq's list: The call can drop its lock and make the pending balance * callback visible to unrelated code paths that call rq_pin_lock(). * - * Just let balance_one() know that it must do it itself. + * Just let dispatch_one() know that it must do it itself. */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) { + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) { rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; return; } /* * No scheduler hooks available. Use the generic irq_work path. The - * above WAKEUP and BALANCE paths should cover most of the cases and the - * time to IRQ re-enable shouldn't be long. + * above WAKEUP and DISPATCH paths should cover most of the cases and + * the time to IRQ re-enable shouldn't be long. */ schedule_deferred(rq); } -static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - u64 reenq_flags, struct rq *locked_rq) +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq) { struct rq *rq; @@ -1223,6 +1094,18 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq if (dsq->id == SCX_DSQ_LOCAL) { rq = container_of(dsq, struct rq, scx.local_dsq); + /* + * A sub-sched lacking baseline access on the target cid has no + * business triggering IPIs. The lockless test is fine: slipping + * through right after a revoke is harmless and a wrong denial + * can't happen - if the caller has seen its ownership, so does + * this test. + */ + if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) { + __scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1); + return; + } + struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; @@ -1273,61 +1156,236 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq schedule_deferred(rq); } -static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) -{ - struct scx_sched *root = rcu_dereference_sched(scx_root); +/* + * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero + * word means no request. Otherwise the fields are: + * + * 63 SCX_SLICE_OOB_PENDING, set on every request + * 62-43 lower bits of issuing scheduler's id + * 42-0 requested slice duration in nsecs + * + * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur + * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore + * a request that outlived a task ownership change. + * + * Only the low 20 bits of sch->id are packed, which is enough to make + * collisions practically impossible. A theoretical collision just lets a stale + * request through once. + */ +enum scx_slice_oob_consts { + SCX_SLICE_OOB_DUR_BITS = 43, + SCX_SLICE_OOB_ID_BITS = 64 - SCX_SLICE_OOB_DUR_BITS - 1, - if (WARN_ON_ONCE(!root)) + SCX_SLICE_OOB_DUR_MASK = (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1, + SCX_SLICE_OOB_ID_SHIFT = SCX_SLICE_OOB_DUR_BITS, + SCX_SLICE_OOB_ID_MASK = (1LLU << SCX_SLICE_OOB_ID_BITS) - 1, + SCX_SLICE_OOB_PENDING = 1LLU << 63, +}; + +/* + * Slice and dsq_vtime write rules + * + * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are + * protected by the rq lock. While running, the rq lock is required because + * update_curr_scx() RMWs the slice and the cap check for slice extension is + * only reliable under the rq lock. + * + * While @p is queued on a user DSQ or on the BPF side, the kernel neither + * consumes nor decides on the fields. Synchronizing the writers is the BPF + * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a + * concurrent DSQ insertion commit can race each other and whichever lands last + * wins. + * + * A DSQ insert kfunc doesn't update the fields directly. The verdict carries + * the values and apply_slice_vtime() commits them at the insertion. + * + * scx_bpf_task_set_slice() may be called from any context and writes directly + * only if @p's rq lock is already held, otherwise it bounces through + * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. + * + * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is + * refused. See set_task_slice_keep_oob(). + * + * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so + * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct + * BPF stores from non-cid-form schedulers are outside these rules. + */ + +/* clear a pending slice request */ +static void clear_task_slice_oob(struct task_struct *p) +{ + if (unlikely(atomic64_read(&p->scx.slice_oob))) + atomic64_set(&p->scx.slice_oob, 0); +} + +/** + * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED + * @dsq: DSQ to insert into + * @p: task being inserted + * + * A HEAD insert should land behind any leading protected tasks. Return %true + * indicates whether @p became the first entry. + */ +static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p) +{ + struct list_head *pos = &dsq->list; + struct scx_dsq_list_node *node; + + /* + * Only rq-owned DSQs can hold protected tasks and the associated rq + * lock keeps their flags stable. + */ + if (!dsq_is_rq_owned(dsq)) { + list_add(&p->scx.dsq_list.node, &dsq->list); + return true; + } + + list_for_each_entry(node, &dsq->list, node) { + struct task_struct *q; + + if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR)) + continue; + + q = container_of(node, struct task_struct, scx.dsq_list); + if (!(q->scx.flags & SCX_TASK_PROTECTED)) + break; + + pos = &node->node; + } + + list_add(&p->scx.dsq_list.node, pos); + + return pos == &dsq->list; +} + +/** + * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request + * @p: task of interest + * @slice: slice to set + * + * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice. + * Refuse and return %false. + */ +static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice) +{ + lockdep_assert_rq_held(task_rq(p)); + + if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + return false; + + p->scx.slice = slice; + return true; +} + +/* set @p's slice, superseding any pending out-of-band request */ +bool scx_set_task_slice(struct task_struct *p, u64 slice) +{ + if (!set_task_slice_keep_oob(p, slice)) + return false; + clear_task_slice_oob(p); + return true; +} + +/** + * scx_task_slice_ended - @p's slice is consumed or given up + * @rq: rq @p is on + * @p: task of interest + * + * End what rides on the slice - the protection, and the rescue if @p is being + * rescued. + * + * A dequeue normally ends the slice too. The exception is a save/restore pair + * on the running task. Attribute changes like renice cycle the task through + * dequeue and enqueue while it keeps executing, so the slice continues. A + * queued task instead loses its DSQ position on any dequeue and the slice ends + * with it. + */ +void scx_task_slice_ended(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + p->scx.flags &= ~SCX_TASK_PROTECTED; + if (unlikely(p == scx_rescuee(rq))) + scx_rescue_end(rq); +} + +/* request @p's slice to be set to @slice, see the write rules above */ +static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) +{ + u64 dur; + + if (slice == SCX_SLICE_INF) { + dur = SCX_SLICE_OOB_DUR_MASK; + } else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) { + dur = SCX_SLICE_OOB_DUR_MASK - 1; + scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1); + } else { + dur = slice; + } + + atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING | + ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur); +} + +/* + * Apply a pending out-of-band slice request under @rq's lock. A request whose + * packed id no longer matches @p's current owner is dropped. An extension needs + * baseline cpu access on @p's cid, shortening is always allowed, and a + * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See + * the write rules above. + */ +static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) +{ + u64 oob, dur, slice; + + lockdep_assert_rq_held(rq); + + if (likely(!atomic64_read(&p->scx.slice_oob))) return; - schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); + oob = atomic64_xchg(&p->scx.slice_oob, 0); + if (unlikely(!oob)) + return; + + /* the issuing scheduler no longer owns @p, drop the request */ + if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) != + (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK))) + return; + + dur = oob & SCX_SLICE_OOB_DUR_MASK; + slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; + + if (slice > p->scx.slice && + unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); + return; + } + + if (unlikely(!set_task_slice_keep_oob(p, slice))) + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); } -/** - * touch_core_sched - Update timestamp used for core-sched task ordering - * @rq: rq to read clock from, must be locked - * @p: task to update the timestamp for - * - * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to - * implement global or local-DSQ FIFO ordering for core-sched. Should be called - * when a task becomes runnable and its turn on the CPU ends (e.g. slice - * exhaustion). +/* + * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them + * and they are committed here, at the insertion. A zero @slice keeps the + * current value, floored at 1 so the task isn't treated as expired. */ -static void touch_core_sched(struct rq *rq, struct task_struct *p) +static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - lockdep_assert_rq_held(rq); + if (slice) { + p->scx.slice = slice; + /* + * An explicit slice supersedes a pending oob request. A carried + * default refill is not an explicit request and must keep it. + */ + if (!(enq_flags & SCX_ENQ_SLICE_DFL)) + clear_task_slice_oob(p); + } else if (!p->scx.slice) { + p->scx.slice = 1; + } -#ifdef CONFIG_SCHED_CORE - /* - * It's okay to update the timestamp spuriously. Use - * sched_core_disabled() which is cheaper than enabled(). - * - * As this is used to determine ordering between tasks of sibling CPUs, - * it may be better to use per-core dispatch sequence instead. - */ - if (!sched_core_disabled()) - p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); -#endif -} - -/** - * touch_core_sched_dispatch - Update core-sched timestamp on dispatch - * @rq: rq to read clock from, must be locked - * @p: task being dispatched - * - * If the BPF scheduler implements custom core-sched ordering via - * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO - * ordering within each local DSQ. This function is called from dispatch paths - * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. - */ -static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) - touch_core_sched(rq, p); -#endif + if (enq_flags & SCX_ENQ_DSQ_PRIQ) + p->scx.dsq_vtime = vtime; } static void update_curr_scx(struct rq *rq) @@ -1335,15 +1393,18 @@ static void update_curr_scx(struct rq *rq) struct task_struct *curr = rq->curr; s64 delta_exec; + /* apply even on 0 delta_exec, callers may still act on the slice */ + apply_task_slice_oob(rq, curr); + delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) { + if (curr->scx.slice != SCX_SLICE_INF) curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); - if (!curr->scx.slice) - touch_core_sched(rq, curr); - } + + if (unlikely(curr == scx_rescuee(rq))) + scx_rescue_charge(rq, delta_exec); dl_server_update(&rq->ext_server, delta_exec); } @@ -1369,8 +1430,8 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * to the CPU or dequeued. In both cases, the only way @p can go back to * the BPF sched is through enqueueing. If being inserted into a local * DSQ with IMMED, persist the state until the next enqueueing event in - * do_enqueue_task() so that we can maintain IMMED protection through - * e.g. SAVE/RESTORE cycles and slice extensions. + * scx_do_enqueue_task() so that we can maintain IMMED protection + * through e.g. SAVE/RESTORE cycles and slice extensions. */ if (enq_flags & SCX_ENQ_IMMED) { if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { @@ -1393,7 +1454,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * done yet, @p can't go on the CPU immediately. Re-enqueue. */ if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } } @@ -1415,7 +1476,11 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { - p->scx.slice = READ_ONCE(sch->slice_dfl); + /* + * A default refill is not an explicit request, so it must not drop a + * pending out-of-band one, which is applied when @p next runs. + */ + set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } @@ -1450,13 +1515,22 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, p->scx.flags &= ~SCX_TASK_IN_CUSTODY; } -static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - struct task_struct *p, u64 enq_flags) +static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - call_task_dequeue(sch, rq, p, 0); + /* + * Only local inserts get the wakeup treatment below. Rejects kick the + * deferred reenq and rescue parks are paced by the rescue timer. + */ + if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { + if (dsq->id == SCX_DSQ_REJECT) + schedule_deferred_locked(rq); + return; + } + /* * Note that @rq's lock may be dropped between this enqueue and @p * actually getting on CPU. This gives higher-class tasks (e.g. RT) @@ -1493,34 +1567,41 @@ static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq wakeup_preempt(rq, p, 0); /* - * If @rq is in balance, the CPU is already vacant and looking for the + * If @rq is in dispatch, the CPU is already vacant and looking for the * next task to run. No need to preempt or trigger resched after moving * @p into its local DSQ. * Note that the wakeup_preempt() above may have already triggered * a resched if @rq->next_class was idle. It's harmless, since * need_resched is cleared immediately after task pick. */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - rq->curr->scx.slice = 0; - resched_curr(rq); + if (likely(scx_set_task_slice(rq->curr, 0))) + resched_curr(rq); + else + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); } } -static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) +static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 slice, u64 vtime, u64 enq_flags) { - bool is_local = dsq->id == SCX_DSQ_LOCAL; + bool is_rq_owned = false; + + if (dsq->id == SCX_DSQ_LOCAL) { + dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); + is_rq_owned = true; + } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || !RB_EMPTY_NODE(&p->scx.dsq_priq)); - if (!is_local) { + if (!is_rq_owned) { raw_spin_lock_nested(&dsq->lock, (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); @@ -1546,6 +1627,13 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, enq_flags &= ~SCX_ENQ_DSQ_PRIQ; } + /* + * @dsq is locked and @enq_flags is sanitized. Commit the carried slice + * and vtime before the PRIQ insertion below reads the new dsq_vtime. + */ + if (enq_flags & SCX_ENQ_APPLY_SLICE) + apply_slice_vtime(p, slice, vtime, enq_flags); + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { struct rb_node *rbp; @@ -1585,9 +1673,8 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, dsq->id); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { - list_add(&p->scx.dsq_list.node, &dsq->list); /* new task inserted at head - use fastpath */ - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } else { /* @@ -1615,12 +1702,16 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, * ops_state first, both sides would modify p->scx.flags * concurrently in a non-atomic way. */ - if (is_local) { - local_dsq_post_enq(sch, dsq, p, enq_flags); + if (is_rq_owned) { + rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { /* - * Task on global/bypass DSQ: leave custody, task on - * non-terminal DSQ: enter custody. + * Global and bypass DSQs are terminal - the task leaves the + * scheduler's custody, so ops.dequeue() fires here. It can run + * without @p's rq lock (finish_dispatch() passes the dispatch + * rq); that's safe because dequeue_task_scx() waits on + * SCX_OPSS_DISPATCHING (see the ops_state note above) and so + * can't race it. A non-terminal DSQ keeps the task in custody. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) call_task_dequeue(sch, rq, p, 0); @@ -1638,8 +1729,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); } -static void task_unlink_from_dsq(struct task_struct *p, - struct scx_dispatch_q *dsq) +void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) { WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); @@ -1652,7 +1742,7 @@ static void task_unlink_from_dsq(struct task_struct *p, list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) { struct task_struct *first_task; first_task = nldsq_next_task(dsq, NULL, false); @@ -1660,10 +1750,10 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void dispatch_dequeue(struct rq *rq, struct task_struct *p) +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; - bool is_local = dsq == &rq->scx.local_dsq; + bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); lockdep_assert_rq_held(rq); @@ -1687,7 +1777,7 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) return; } - if (!is_local) + if (!is_rq_owned) raw_spin_lock(&dsq->lock); /* @@ -1696,7 +1786,7 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) */ if (p->scx.holding_cpu < 0) { /* @p must still be on @dsq, dequeue */ - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); } else { /* * We're racing against dispatch_to_local_dsq() which already @@ -1709,13 +1799,13 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) } p->scx.dsq = NULL; - if (!is_local) + if (!is_rq_owned) raw_spin_unlock(&dsq->lock); } /* - * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq - * and dsq are locked. + * Abbreviated version of scx_dispatch_dequeue() that can be used when both + * @p's rq and dsq are locked. */ static void dispatch_dequeue_locked(struct task_struct *p, struct scx_dispatch_q *dsq) @@ -1723,7 +1813,7 @@ static void dispatch_dequeue_locked(struct task_struct *p, lockdep_assert_rq_held(task_rq(p)); lockdep_assert_held(&dsq->lock); - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); p->scx.dsq = NULL; } @@ -1750,6 +1840,10 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, else dsq = find_user_dsq(sch, dsq_id); + /* + * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE + * hit the error below. They cannot be reached with an ID. + */ if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); return find_global_dsq(sch, tcpu); @@ -1761,7 +1855,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, static void mark_direct_dispatch(struct scx_sched *sch, struct task_struct *ddsp_task, struct task_struct *p, u64 dsq_id, - u64 enq_flags) + u64 slice, u64 vtime, u64 enq_flags) { /* * Mark that dispatch already happened from ops.select_cpu() or @@ -1785,6 +1879,8 @@ static void mark_direct_dispatch(struct scx_sched *sch, WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); WARN_ON_ONCE(p->scx.ddsp_enq_flags); + p->scx.ddsp_slice = slice; + p->scx.ddsp_vtime = vtime; p->scx.ddsp_dsq_id = dsq_id; p->scx.ddsp_enq_flags = enq_flags; } @@ -1796,10 +1892,10 @@ static void mark_direct_dispatch(struct scx_sched *sch, * - direct_dispatch(): cleared on the synchronous enqueue path, deferred * dispatch keeps the state until consumed * - process_ddsp_deferred_locals(): cleared after consuming deferred state, - * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch * verdict is ignored (local/global/bypass) - * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred - * cancellation and holding_cpu races + * - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering + * deferred cancellation and holding_cpu races * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by * the scx_bypass() loop, so that stale state is not reused by a subsequent * scheduler instance @@ -1816,9 +1912,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); - u64 ddsp_enq_flags; - - touch_core_sched_dispatch(rq, p); + u64 ddsp_enq_flags, slice, vtime; p->scx.ddsp_enq_flags |= enq_flags; @@ -1858,12 +1952,15 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, } ddsp_enq_flags = p->scx.ddsp_enq_flags; + slice = p->scx.ddsp_slice; + vtime = p->scx.ddsp_vtime; clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } -static bool scx_rq_online(struct rq *rq) +bool scx_rq_online(struct rq *rq) { /* * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates @@ -1875,8 +1972,8 @@ static bool scx_rq_online(struct rq *rq) return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } -static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) { struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; @@ -1897,6 +1994,24 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ p->scx.flags &= ~SCX_TASK_IMMED; + /* + * A task reenqueued too many times without running means the scheduler + * keeps re-deciding a placement it can't honor, e.g. re-inserting to a + * cid it lacks caps on. Eject the owning scheduler and strand the task + * to be picked up during sched exit. + */ + if (enq_flags & SCX_ENQ_REENQ) { + if (++p->scx.reenq_cnt > 1) + __scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1); + + if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) { + __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), + "%s[%d] reenqueued %u times without running", + p->comm, p->pid, p->scx.reenq_cnt); + return; + } + } + /* * If !scx_rq_online(), we already told the BPF scheduler that the CPU * is offline and are just running the hotplug path. Don't bother the @@ -1917,6 +2032,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && unlikely(p->flags & PF_EXITING)) { __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ goto local; } @@ -1963,7 +2079,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, direct_dispatch(sch, p, enq_flags); return; local_norefill: - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); return; local: dsq = &rq->scx.local_dsq; @@ -1976,15 +2092,9 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, goto enqueue; enqueue: - /* - * For task-ordering, slice refill must be treated as implying the end - * of the current slice. Otherwise, the longer @p stays on the CPU, the - * higher priority it becomes from scx_prio_less()'s POV. - */ - touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -2006,20 +2116,30 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p) * appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); + + /* + * Record the rq @p is runnable on, maintained under the rq lock so it + * stays valid unlike task_cpu(), which a remote wakeup can move under + * pi_lock alone. + */ + WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq)); } static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); - if (reset_runnable_at) + WRITE_ONCE(p->scx.runnable_cpu, -1); + if (reset_runnable_at) { p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + p->scx.reenq_cnt = 0; + } } static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) { struct scx_sched *sch = scx_task_sched(p); int sticky_cpu = p->scx.sticky_cpu; - u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; + u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; @@ -2028,10 +2148,13 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ * Restoring a running task will be immediately followed by * set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force - * direct-dispatch into the local DSQ by setting the sticky_cpu. + * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark + * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) + if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { sticky_cpu = cpu_of(rq); + enq_flags |= SCX_ENQ_IGNORE_CAPS; + } if (p->scx.flags & SCX_TASK_QUEUED) { WARN_ON_ONCE(!task_runnable(p)); @@ -2046,14 +2169,11 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); - if (enq_flags & SCX_ENQ_WAKEUP) - touch_core_sched(rq, p); - /* Start dl_server if this is the first task being enqueued */ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server); - do_enqueue_task(rq, p, enq_flags, sticky_cpu); + scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; @@ -2148,9 +2268,9 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ /* * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property - * change (not sleep or core-sched pick). + * change (not sleep). */ - if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) + if (!(deq_flags & DEQUEUE_SLEEP)) deq_flags |= SCX_DEQ_SCHED_CHANGE; if (!(p->scx.flags & SCX_TASK_QUEUED)) { @@ -2168,13 +2288,15 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while - * balance_one() leaves @rq unlocked. However, they don't contain any + * dispatch_one() leaves @rq unlocked. However, they don't contain any * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { + if (task_current(rq, p) && + (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { update_curr_scx(rq); - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -2189,7 +2311,12 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ rq->scx.nr_running--; sub_nr_running(rq, 1); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); + + /* see scx_task_slice_ended() for the save/restore exception */ + if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + scx_task_slice_ended(rq, p); + clear_direct_dispatch(p); return true; } @@ -2199,10 +2326,13 @@ static void yield_task_scx(struct rq *rq) struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* a yield gives the slice up */ + scx_task_slice_ended(rq, p); + if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else - p->scx.slice = 0; + scx_set_task_slice(p, 0); } static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) @@ -2210,6 +2340,9 @@ static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) struct task_struct *from = rq->donor; struct scx_sched *sch = scx_task_sched(from); + /* like a plain yield, giving the slice up ends the protection */ + scx_task_slice_ended(rq, from); + if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); else @@ -2237,35 +2370,36 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl * - A higher-priority wakes up while SCX dispatch is in progress. */ if (rq->scx.nr_immed) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } -static void move_local_task_to_local_dsq(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) +void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags, struct scx_dispatch_q *src_dsq, + struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; + struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @dsq is locked and @p is on @dst_rq */ - lockdep_assert_held(&src_dsq->lock); + /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ + if (!dsq_is_rq_owned(src_dsq)) + lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) - list_add(&p->scx.dsq_list.node, &dst_dsq->list); + dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); dsq_inc_nr(dst_dsq, p, enq_flags); p->scx.dsq = dst_dsq; - local_dsq_post_enq(sch, dst_dsq, p, enq_flags); + rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags); } /** * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ + * @sch: scheduler placing @p * @p: task to move * @enq_flags: %SCX_ENQ_* * @src_rq: rq to move the task from, locked on entry, released on return @@ -2273,7 +2407,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, * * Move @p which is currently on @src_rq to @dst_rq's local DSQ. */ -static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, +static void move_remote_task_to_local_dsq(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { lockdep_assert_rq_held(src_rq); @@ -2289,15 +2424,19 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, switch_rq_lock(src_rq, dst_rq); /* - * We want to pass scx-specific enq_flags but activate_task() will - * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.extra_enq_flags instead. + * activate_task() below truncates enq_flags to 32 bits and re-derives + * @p's owner, dropping our scx flags and the placing @sch. We own @rq, + * so stash both across the call. The enqueue reads them back, keeping + * the scx flags and checking caps against the placer, not the owner. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); - dst_rq->scx.extra_enq_flags = enq_flags; + WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || + dst_rq->scx.remote_activate_sch); + dst_rq->scx.remote_activate_enq_flags = enq_flags; + dst_rq->scx.remote_activate_sch = sch; activate_task(dst_rq, p, 0); - dst_rq->scx.extra_enq_flags = 0; + dst_rq->scx.remote_activate_enq_flags = 0; + dst_rq->scx.remote_activate_sch = NULL; } /* @@ -2378,13 +2517,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, } /** - * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq + * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task * @dsq: locked DSQ @p is currently on + * @locked_rq: currently locked rq * @src_rq: rq @p is currently on, stable with @dsq locked * - * Called with @dsq locked but no rq's locked. We want to move @p to a different - * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is + * Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ, + * including any local DSQ, but are not locking @src_rq. Locking @src_rq is * required when transferring into a local DSQ. Even when transferring into a * non-local DSQ, it's better to use the same mechanism to protect against * dequeues and maintain the invariant that @p->scx.dsq can only change while @@ -2401,43 +2541,42 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still - * points to this CPU. See dispatch_dequeue() for the counterpart. + * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. */ -static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, - struct scx_dispatch_q *dsq, - struct rq *src_rq) +static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, + struct scx_dispatch_q *dsq, + struct rq *locked_rq, + struct rq *src_rq) { s32 cpu = raw_smp_processor_id(); lockdep_assert_held(&dsq->lock); + lockdep_assert_rq_held(locked_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); p->scx.holding_cpu = cpu; raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(src_rq); + switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ return likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p)); } -static bool consume_remote_task(struct rq *this_rq, +static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { - raw_spin_rq_unlock(this_rq); - - if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { - move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); + if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(this_rq); + switch_rq_lock(src_rq, this_rq); return false; } } @@ -2489,14 +2628,12 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, if (dst_dsq->id == SCX_DSQ_LOCAL) { /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { - task_unlink_from_dsq(p, src_dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, - src_dsq, dst_rq); + scx_task_unlink_from_dsq(p, src_dsq); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } } else { /* @@ -2506,14 +2643,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); - dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags); } return dst_rq; } -static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) { struct task_struct *p; retry: @@ -2542,14 +2679,14 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, break; if (rq == task_rq) { - task_unlink_from_dsq(p, dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); + scx_task_unlink_from_dsq(p, dsq); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); return true; } if (task_can_run_on_remote_rq(sch, p, rq, false)) { - if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) + if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) return true; goto retry; } @@ -2559,11 +2696,11 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, return false; } -static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) { int node = cpu_to_node(cpu_of(rq)); - return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); + return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); } /** @@ -2572,6 +2709,8 @@ static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) * @rq: current rq which is locked * @dst_dsq: destination DSQ * @p: task to dispatch + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local @@ -2582,8 +2721,8 @@ static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) * %SCX_OPSS_DISPATCHING). */ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dst_dsq, - struct task_struct *p, u64 enq_flags) + struct scx_dispatch_q *dst_dsq, struct task_struct *p, + u64 slice, u64 vtime, u64 enq_flags) { struct rq *src_rq = task_rq(p); struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); @@ -2596,8 +2735,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { - dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); return; } @@ -2610,7 +2749,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * As DISPATCHING guarantees that @p is wholly ours, we can pretend that * we're moving from a DSQ and use the same mechanism - mark the task * under transfer with holding_cpu, release DISPATCHING and then follow - * the same protocol. See unlink_dsq_and_lock_src_rq(). + * the same protocol. See unlink_dsq_and_switch_rq_lock(). */ p->scx.holding_cpu = raw_smp_processor_id(); @@ -2634,16 +2773,18 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, */ if (src_rq == dst_rq) { p->scx.holding_cpu = -1; - dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); + scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, + slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; - dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), - p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); + scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), + p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | + SCX_ENQ_GDSQ_FALLBACK); } else { - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + apply_slice_vtime(p, slice, vtime, enq_flags); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; } @@ -2660,6 +2801,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, /** * finish_dispatch - Asynchronously finish dispatching a task + * @sch: the scheduler * @rq: current rq which is locked * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched @@ -2677,15 +2819,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * was valid in the first place. Make sure that the task is still owned by the * BPF scheduler and claim the ownership before dispatching. */ -static void finish_dispatch(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, - unsigned long qseq_at_dispatch, - u64 dsq_id, u64 enq_flags) +static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, + unsigned long qseq_at_dispatch, u64 dsq_id, + u64 slice, u64 vtime, u64 enq_flags) { struct scx_dispatch_q *dsq; unsigned long opss; - touch_core_sched_dispatch(rq, p); retry: /* * No need for _acquire here. @p is accessed only after a successful @@ -2726,10 +2866,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, goto retry; case SCX_OPSS_QUEUEING: /* - * do_enqueue_task() is in the process of transferring the task - * to the BPF scheduler while holding @p's rq lock. As we aren't - * holding any kernel or BPF resource that the enqueue path may - * depend upon, it's safe to wait. + * scx_do_enqueue_task() is in the process of transferring the + * task to the BPF scheduler while holding @p's rq lock. As we + * aren't holding any kernel or BPF resource that the enqueue + * path may depend upon, it's safe to wait. */ wait_ops_state(p, opss); goto retry; @@ -2737,15 +2877,16 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (dsq->id == SCX_DSQ_LOCAL) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); else - dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } -static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; u32 u; @@ -2754,7 +2895,7 @@ static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) struct scx_dsp_buf_ent *ent = &dspc->buf[u]; finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, - ent->enq_flags); + ent->slice, ent->vtime, ent->enq_flags); } dspc->nr_tasks += dspc->cursor; @@ -2774,123 +2915,16 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) +static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) { - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (consume_global_dsq(sch, rq)) - return true; - - if (bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, - * the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - - return false; -} - -static int balance_one(struct rq *rq, struct task_struct *prev) -{ - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); + enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); lockdep_assert_rq_held(rq); - rq->scx.flags |= SCX_RQ_IN_BALANCE; - rq->scx.flags &= ~SCX_RQ_BAL_KEEP; + rq->scx.flags |= SCX_RQ_IN_DISPATCH; + + scx_process_sync_ecaps(rq, prev); if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { @@ -2920,16 +2954,19 @@ static int balance_one(struct rq *rq, struct task_struct *prev) */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && !scx_bypassing(sch, cpu)) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; + verdict = SCX_DSP_PREV; goto has_tasks; } } /* if there already are tasks to run, nothing to do */ - if (rq->scx.local_dsq.nr) + if (rq->scx.local_dsq.nr) { + verdict = SCX_DSP_LOCAL; goto has_tasks; + } - if (scx_dispatch_sched(sch, rq, prev, false)) + verdict = scx_dispatch_sched(sch, rq, prev, false); + if (verdict != SCX_DSP_NONE) goto has_tasks; /* @@ -2937,13 +2974,14 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; + (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + scx_task_can_stay_on_cpu(rq, prev)) { __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; goto has_tasks; } - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return false; + rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; + return SCX_DSP_NONE; has_tasks: /* @@ -2957,10 +2995,10 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * between the IMMED queueing and the subsequent scheduling event. */ if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return true; + rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; + return verdict; } static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) @@ -2973,7 +3011,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) * dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); } p->se.exec_start = rq_clock_task(rq); @@ -2984,6 +3022,9 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) clr_task_runnable(p, true); + /* apply any pending out-of-band slice request before the tick decision */ + apply_task_slice_oob(rq, p); + /* * @p is getting newly scheduled or got kicked after someone updated its * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be @@ -3035,7 +3076,7 @@ preempt_reason_from_class(const struct sched_class *class) static void switch_class(struct rq *rq, struct task_struct *next) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); const struct sched_class *next_class = next->sched_class; if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) @@ -3057,7 +3098,7 @@ static void switch_class(struct rq *rq, struct task_struct *next) * preempted, and it regaining control of the CPU. * * ->cpu_release() complements ->cpu_acquire(), which is emitted the - * next time that balance_one() is invoked. + * next time that dispatch_one() is invoked. */ if (!rq->scx.cpu_released) { if (sch->ops.cpu_release) { @@ -3076,12 +3117,24 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool rescue_keep = false; /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); update_curr_scx(rq); + /* + * If the slice is consumed, protection ends with it. A rescuee + * preempted beforehand keeps going, see scx_rescue_keep(). + */ + if (!p->scx.slice) { + if (unlikely(p == scx_rescuee(rq))) + rescue_keep = scx_rescue_keep(rq, p); + if (!rescue_keep) + scx_task_slice_ended(rq, p); + } + /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(sch, stopping, rq, p, true); @@ -3095,14 +3148,34 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. + * + * An open rescue must keep @p on the local DSQ even if the + * scheduler zeroed the slice in ops.stopping() above. */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && + !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + u64 enq_flags = 0; + + /* + * Keep a preempted rescue going. If preempted + * by another SCX task, append to the local DSQ, + * see scx_rescue_keep(). + */ + if (unlikely(p == scx_rescuee(rq))) { + enq_flags |= SCX_ENQ_IGNORE_CAPS; + if (!rescue_keep) + enq_flags |= SCX_ENQ_HEAD; + } else { + enq_flags |= SCX_ENQ_HEAD; + } + + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); } goto switch_class; } @@ -3112,17 +3185,19 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. + * This doesn't apply if the baseline access on the CPU is lost. * - * Core scheduling can force this CPU idle while @p stays - * runnable. @p's cookie then won't match the core's, so skip - * the warning in that case. + * Under core scheduling, a pick dispatches only when nothing is + * locally runnable and can legitimately go idle with @p still + * runnable (see do_pick_task_scx()). */ - if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && + if (next && sched_class_above(&ext_sched_class, next->sched_class) && + scx_task_can_stay_on_cpu(rq, p)) { + WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); - do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { - do_enqueue_task(rq, p, 0, -1); + scx_do_enqueue_task(rq, p, 0, -1); } } @@ -3133,11 +3208,26 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, static void kick_sync_wait_bal_cb(struct rq *rq) { - struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); - unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + struct scx_kick_syncs __rcu *ks; + unsigned long *ksyncs; bool waited; s32 cpu; + /* + * This callback is queued and normally flushed within @rq's own + * scheduling pass. However, dispatch can drop the rq lock while it sits + * queued, and lock takers in that window (the sched class change paths, + * the scx task iterator) flush pending balance callbacks on release, + * running this one on a foreign CPU whose snapshots are unrelated. The + * kicked CPUs are already on their way to advance the kick_syncs being + * waited on. Don't get in the way. + */ + if (unlikely(cpu_of(rq) != smp_processor_id())) + return; + + ks = __this_cpu_read(scx_kick_syncs); + ksyncs = rcu_dereference_sched(ks)->syncs; + /* * Drop rq lock and enable IRQs while waiting. IRQs must be enabled * — a target CPU may be waiting for us to process an IPI (e.g. TLB @@ -3160,6 +3250,7 @@ static void kick_sync_wait_bal_cb(struct rq *rq) continue; } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irq(rq); while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3179,27 +3270,23 @@ static struct task_struct *first_local_task(struct rq *rq) struct task_struct, scx.dsq_list.node); } -static struct task_struct * -do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +/* + * Run dispatch and queue the follow-up work for a pick. + */ +static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) { - struct task_struct *prev = rq->curr; - bool keep_prev; - struct task_struct *p; - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - rq_modified_begin(rq, &ext_sched_class); + enum scx_dsp_verdict verdict; rq_unpin_lock(rq, rf); - balance_one(rq, prev); + verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq); /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + * Defer to a balance callback which can drop rq lock and enable IRQs. + * Waiting directly in the pick path would deadlock against CPUs sending + * us IPIs (e.g. TLB flushes) while we wait for them. */ if (unlikely(rq->scx.kick_sync_pending)) { rq->scx.kick_sync_pending = false; @@ -3207,10 +3294,90 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) kick_sync_wait_bal_cb); } + return verdict; +} + +#ifdef CONFIG_SCHED_CORE +/* + * Dispatch for a pick when core scheduling is enabled. The selection picks for + * all SMT siblings and the rq_i->core_pick state it builds must stay atomic + * throughout. If the dispatch released the rq lock, anything can have happened + * in between - return %SCX_DSP_RETRY to restart the selection against current + * state. + */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + enum scx_dsp_verdict verdict; + u32 seq = rq->scx.lock_drop_seq; + + /* another dispatch is in flight on @rq, let that handle it */ + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) + return SCX_DSP_NONE; + + rq_unpin_lock(rq, rf); + + verdict = dispatch_one(rq, prev); + + if (cpu_of(rq) == smp_processor_id()) { + maybe_queue_balance_callback(rq); + + /* see dispatch_pick() */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + /* + * Balance callbacks must run in the context that queued them, + * so they can't be queued on another CPU's rq. Run the deferred + * work directly instead. + */ + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + run_deferred(rq); + } + + rq_repin_lock(rq, rf); + + /* if dispatch_one() released the rq lock, restart the selection */ + if (rq->scx.lock_drop_seq != seq) + return SCX_DSP_RETRY; + + return verdict; +} +#else /* CONFIG_SCHED_CORE */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + return SCX_DSP_NONE; +} +#endif /* CONFIG_SCHED_CORE */ + +static struct task_struct * +do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +{ + struct task_struct *prev = rq->curr; + enum scx_dsp_verdict verdict; + struct task_struct *p; + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + rq_modified_begin(rq, &ext_sched_class); + + if (sched_core_enabled(rq)) + verdict = dispatch_core_pick(rq, rf, prev); + else + verdict = dispatch_pick(rq, rf, prev); + + if (verdict == SCX_DSP_RETRY) + return RETRY_TASK; + /* - * If any higher-priority sched class enqueued a runnable task on - * this rq during balance_one(), abort and return RETRY_TASK, so - * that the scheduler loop can restart. + * If any higher-priority sched class enqueued a runnable task on this + * rq during dispatch_one(), abort and return RETRY_TASK, so that the + * scheduler loop can restart. * * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. @@ -3218,28 +3385,23 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) if (!force_scx && rq_modified_above(rq, &ext_sched_class)) return RETRY_TASK; - keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; - if (unlikely(keep_prev && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - keep_prev = false; - } - /* - * If balance_one() is telling us to keep running @prev, replenish slice - * if necessary and keep running @prev. Otherwise, pop the first one - * from the local DSQ. + * If we're keeping @prev, replenish slice if necessary and keep running + * @prev. Otherwise, pop the first one from the local DSQ. */ - if (keep_prev) { + if (verdict == SCX_DSP_PREV) { p = prev; - if (!p->scx.slice) + if (!p->scx.slice) { + /* the slice is consumed, protection ends */ + scx_task_slice_ended(rq, p); refill_task_slice_dfl(scx_task_sched(p), p); + } } else { p = first_local_task(rq); if (!p) return NULL; - if (unlikely(!p->scx.slice)) { + if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); if (!scx_bypassing(sch, cpu_of(rq)) && @@ -3298,33 +3460,70 @@ void ext_server_init(struct rq *rq) * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * - * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used - * to implement the default task ordering. The older the timestamp, the higher - * priority the task - the global FIFO ordering matching the default scheduling - * behavior. + * A pair of tasks owned by one scheduler is ordered by the owner's + * ops.core_sched_before(). A pair spanning two schedulers is ordered by their + * nearest common ancestor which implements the op - the one case where the op + * is called on tasks that the scheduler delegated to its sub-schedulers and may + * not be scheduling anymore. * - * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to - * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * When neither applies, or the deciding scheduler is bypassing on either task's + * CPU, the default ordering runs the task which has been waiting longer first. + * A running task counts as the most recently serviced and orders after every + * waiting task. Waiting tasks are compared by @p->scx.runnable_at. + * + * Return: %true if @a should run after @b. */ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi) { struct scx_sched *sch_a = scx_task_sched(a); struct scx_sched *sch_b = scx_task_sched(b); + struct scx_sched *sch = NULL; + bool a_running, b_running; + + if (sch_a == sch_b) { + if (SCX_HAS_OP(sch_a, core_sched_before)) + sch = sch_a; + } else { + s32 level; + + for (level = min(sch_a->level, sch_b->level); level >= 0; level--) { + struct scx_sched *anc = sch_a->ancestors[level]; + + if (anc == sch_b->ancestors[level] && + SCX_HAS_OP(anc, core_sched_before)) { + sch = anc; + break; + } + } + } /* + * scx_prio_less() returns whether @a should run after @b while + * ops.core_sched_before() returns whether its first argument should run + * before the second. Swap the arguments. + * * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. */ - if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && - !scx_bypassing(sch_a, task_cpu(a))) - return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, - task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); - else - return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); + if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b))) + return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a), + (struct task_struct *)b, + (struct task_struct *)a); + + /* + * runnable_at is refreshed only on enqueue, so a task which keeps + * occupying its CPU carries a stale stamp. A running task is the most + * recently serviced whatever its stamp says. Order it after every + * waiting task. + */ + a_running = a->on_cpu; + b_running = b->on_cpu; + if (a_running != b_running) + return a_running; + + return time_after(a->scx.runnable_at, b->scx.runnable_at); } #endif /* CONFIG_SCHED_CORE */ @@ -3369,17 +3568,34 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag } else { s32 cpu; + /* + * While bypassing, the enqueue path routes @p to a bypass DSQ + * without consulting the direct-dispatch target, making the + * default selection pointless. It doesn't work anyway when the + * scheduler does its own idle tracking and the built-in idle + * cpumasks are not updated. Leave @p on @prev_cpu. + */ + if (bypassing) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + p->scx.selected_cpu = prev_cpu; + return prev_cpu; + } + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); if (cpu >= 0) { - refill_task_slice_dfl(sch, p); + /* + * Carry the slice refill and let the insertion commit + * it under rq lock. See the write rules. + */ + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); + p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); + p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL; p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; } else { cpu = prev_cpu; } p->scx.selected_cpu = cpu; - if (bypassing) - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); return cpu; } } @@ -3413,8 +3629,9 @@ static void set_cpus_allowed_scx(struct task_struct *p, static void handle_hotplug(struct rq *rq, bool online) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected(); s32 cpu = cpu_of(rq); + s32 cpu_or_cid = cpu; atomic_long_inc(&scx_hotplug_seq); @@ -3429,10 +3646,31 @@ static void handle_hotplug(struct rq *rq, bool online) if (scx_enabled()) scx_idle_update_selcpu_topology(&sch->ops); + if (online) + scx_online_ecaps(rq); + else + scx_offline_ecaps(rq); + + /* + * The tables can't be retired while this function is running as the + * retirement is inside cpus_read_lock. However, scx_cpu_arg() is + * awkward here as the tables can be NULL after root enable failure and + * lockdep would trigger without surrounding rcu_read_lock(). Open code + * the translation. If the table is NULL, the ops are also cleared and + * @cpu_or_cid goes unused. + */ + if (scx_is_cid_type()) { + s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, + lockdep_is_cpus_held()); + + if (tbl) + cpu_or_cid = tbl[cpu]; + } + if (online && SCX_HAS_OP(sch, cpu_online)) - SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); + SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); else if (!online && SCX_HAS_OP(sch, cpu_offline)) - SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); + SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid); else scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, @@ -3458,6 +3696,7 @@ static void rq_online_scx(struct rq *rq) static void rq_offline_scx(struct rq *rq) { rq->scx.flags &= ~SCX_RQ_ONLINE; + scx_rescue_flush(rq); } static bool check_rq_for_timeouts(struct rq *rq) @@ -3478,8 +3717,19 @@ static bool check_rq_for_timeouts(struct rq *rq) if (unlikely(time_after(jiffies, last_runnable + READ_ONCE(sch->watchdog_timeout)))) { + struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); + /* + * A task can be stuck on a DSQ that a sched other than + * its owner is responsible for draining, e.g. an + * ancestor's bypass DSQ while the owner is bypassing. + * Blame the drainer. The local DSQ is consumed by the + * cpu itself and keeps blame on the owner. + */ + if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL) + sch = dsq->sched; + __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, @@ -3544,15 +3794,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) update_curr_scx(rq); /* - * While disabling, always resched and refresh core-sched timestamp as - * we can't trust the slice management or ops.core_sched_before(). + * While disabling, always resched as we can't trust the slice + * management. */ - if (scx_bypassing(sch, cpu_of(rq))) { - curr->scx.slice = 0; - touch_core_sched(rq, curr); - } else if (SCX_HAS_OP(sch, tick)) { + if (scx_bypassing(sch, cpu_of(rq))) + scx_set_task_slice(curr, 0); + else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, curr); - } if (!curr->scx.slice) resched_curr(rq); @@ -3572,15 +3820,28 @@ static struct cgroup *tg_cgrp(struct task_group *tg) return &cgrp_dfl_root.cgrp; } -#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), +#define SCX_INIT_TASK_ARGS_CGROUP(cgrp) .cgroup = (cgrp), #else /* CONFIG_EXT_GROUP_SCHED */ -#define SCX_INIT_TASK_ARGS_CGROUP(tg) +#define SCX_INIT_TASK_ARGS_CGROUP(cgrp) #endif /* CONFIG_EXT_GROUP_SCHED */ -static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +/** + * __scx_init_task - Initialize a task for a sched + * @sch: sched to initialize @p for + * @p: task of interest + * @cgrp: cgroup @p is joining, %NULL for @p's current task_group's cgroup + * @fork: %true if @p is being forked + * + * Pre-commit cgroup migration passes @cgrp explicitly as @p's task_group + * still reflects the source. + * + * Return 0 on success, -errno on failure. + */ +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, + struct cgroup *cgrp, bool fork) { int ret; @@ -3588,13 +3849,13 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo if (SCX_HAS_OP(sch, init_task)) { struct scx_init_task_args args = { - SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) + SCX_INIT_TASK_ARGS_CGROUP(cgrp ?: tg_cgrp(task_group(p))) .fork = fork, }; ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); if (unlikely(ret)) { - ret = ops_sanitize_err(sch, "init_task", ret); + ret = scx_ops_sanitize_err(sch, "init_task", ret); return ret; } } @@ -3666,7 +3927,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); } -static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) +void scx_enable_task(struct scx_sched *sch, struct task_struct *p) { __scx_enable_task(sch, p); scx_set_task_state(p, SCX_TASK_ENABLED); @@ -3690,7 +3951,9 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; - p->scx.slice = 0; + scx_task_slice_ended(rq, p); + scx_set_task_slice(p, 0); + p->scx.reenq_cnt = 0; /* * Verify the task is not in BPF scheduler's custody. If flag @@ -3700,8 +3963,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); } -static void __scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = false, @@ -3735,19 +3997,19 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch, * ran. The task state has not been transitioned, so this mirrors the * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). */ -static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = true }; lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); + /* @p was never associated with @sch, dispatch on the explicit @sch */ if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); + __SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args); } -static void scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { __scx_disable_and_exit_task(sch, p); @@ -3775,6 +4037,7 @@ void init_scx_entity(struct sched_ext_entity *scx) RB_CLEAR_NODE(&scx->dsq_priq); scx->sticky_cpu = -1; scx->holding_cpu = -1; + scx->runnable_cpu = -1; INIT_LIST_HEAD(&scx->runnable_node); scx->runnable_at = jiffies; scx->ddsp_dsq_id = SCX_DSQ_INVALID; @@ -3829,12 +4092,12 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) if (scx_init_task_enabled) { #ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; + struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp); #else - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); #endif scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - ret = __scx_init_task(sch, p, true); + ret = __scx_init_task(sch, p, NULL, true); if (unlikely(ret)) { scx_set_task_state(p, SCX_TASK_NONE); return ret; @@ -3877,7 +4140,7 @@ void scx_post_fork(struct task_struct *p) void scx_cancel_fork(struct task_struct *p) { - if (scx_enabled()) { + if (scx_init_task_enabled) { struct rq *rq; struct rq_flags rf; @@ -4058,13 +4321,15 @@ static void process_ddsp_deferred_locals(struct rq *rq) struct scx_dispatch_q *dsq; u64 dsq_id = p->scx.ddsp_dsq_id; u64 enq_flags = p->scx.ddsp_enq_flags; + u64 slice = p->scx.ddsp_slice; + u64 vtime = p->scx.ddsp_vtime; list_del_init(&p->scx.dsq_list.node); clear_direct_dispatch(p); dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); } } @@ -4088,16 +4353,20 @@ static void process_ddsp_deferred_locals(struct rq *rq) * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers - * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT - * in process_deferred_reenq_locals(). + * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in + * scx_do_enqueue_task(), which ejects the task's owning scheduler. */ -static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) +static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, + u64 *reenq_flags, u32 *reason) { bool first; first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) + return false; + *reason = SCX_TASK_REENQ_KFUNC; if ((p->scx.flags & SCX_TASK_IMMED) && @@ -4107,6 +4376,12 @@ static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 return true; } + if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) && + scx_task_reenq_on_cap_revoke(rq, p)) { + *reason = SCX_TASK_REENQ_CAP; + return true; + } + return *reenq_flags & SCX_REENQ_ANY; } @@ -4150,10 +4425,10 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!scx_is_descendant(task_sch, sch)) continue; - if (!local_task_should_reenq(p, &reenq_flags, &reason)) + if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; @@ -4165,25 +4440,41 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { list_del_init(&p->scx.dsq_list.node); - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } + /* + * The revoke that scheduled this scan may have raced the pick: curr + * may be a now-capless task, either one that kept running or one + * promoted off the local DSQ between the ecaps sync and this scan. + * Zero the slice to evict it. The enqueue gate blocks new capless + * inserts, so no later pick can slip through after the scan. + */ + if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && + rq->curr->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->curr)) { + scx_set_task_slice(rq->curr, 0); + resched_curr(rq); + } + return nr_enqueued; } static void process_deferred_reenq_locals(struct rq *rq) { - u64 seq = ++rq->scx.deferred_reenq_locals_seq; - lockdep_assert_rq_held(rq); + /* + * A task can be re-queued within this loop when a reenqueued task + * bounces straight back to the local DSQ. That recursion is bounded by + * the per-task reenqueue cap in scx_do_enqueue_task(). + */ while (true) { struct scx_sched *sch; u64 reenq_flags; - bool skip = false; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_local *drl = @@ -4202,27 +4493,12 @@ static void process_deferred_reenq_locals(struct rq *rq) reenq_flags = drl->flags; WRITE_ONCE(drl->flags, 0); list_del_init(&drl->node); - - if (likely(drl->seq != seq)) { - drl->seq = seq; - drl->cnt = 0; - } else { - if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { - scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", - drl->cnt); - skip = true; - } - - __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); - } } - if (!skip) { - /* see schedule_dsq_reenq() */ - smp_mb(); + /* see schedule_dsq_reenq() */ + smp_mb(); - reenq_local(sch, rq, reenq_flags); - } + reenq_local(sch, rq, reenq_flags); } } @@ -4258,8 +4534,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag task_rq = task_rq(p); if (locked_rq != task_rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } if (unlikely(!raw_spin_rq_trylock(task_rq))) { raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(task_rq); @@ -4280,11 +4558,12 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; p->scx.flags |= reason; - do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); locked_rq = NULL; cpu_relax(); @@ -4297,8 +4576,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag raw_spin_unlock(&dsq->lock); if (locked_rq != rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } raw_spin_rq_lock(rq); } } @@ -4309,7 +4590,7 @@ static void process_deferred_reenq_users(struct rq *rq) while (true) { struct scx_dispatch_q *dsq; - u64 reenq_flags; + u64 dsq_id, reenq_flags; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_user *dru = @@ -4332,7 +4613,12 @@ static void process_deferred_reenq_users(struct rq *rq) /* see schedule_dsq_reenq() */ smp_mb(); - BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); + /* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */ + dsq_id = READ_ONCE(dsq->id); + if (unlikely(dsq_id == SCX_DSQ_INVALID)) + continue; + + BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN); reenq_user(rq, dsq, reenq_flags); } } @@ -4346,6 +4632,8 @@ static void run_deferred(struct rq *rq) if (!list_empty(&rq->scx.deferred_reenq_users)) process_deferred_reenq_users(rq); + + scx_reenq_reject(rq); } #ifdef CONFIG_NO_HZ_FULL @@ -4369,6 +4657,13 @@ bool scx_can_stop_tick(struct rq *rq) if (scx_bypassing(sch, cpu_of(rq))) return false; + /* + * A running rescuee's charging and expiry are tick-driven, see + * scx_rescue_charge(). Keep the tick while rescue is in progress. + */ + if (unlikely(p == scx_rescuee(rq))) + return false; + /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks @@ -4381,7 +4676,6 @@ bool scx_can_stop_tick(struct rq *rq) #ifdef CONFIG_EXT_GROUP_SCHED DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); -static bool scx_cgroup_enabled; void scx_tg_init(struct task_group *tg) { @@ -4391,14 +4685,80 @@ void scx_tg_init(struct task_group *tg) tg->scx.idle = false; } +/** + * scx_tg_sched - Resolve a task_group's sched + * @tg: task_group of interest + * + * Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg + * isn't inited. An autogroup tg has no cgroup of its own and resolves to the + * root sched. + * + * When a child sched exits, its task_groups are moved to the parent and + * re-inited on it. A failed re-init fails the parent in turn and leaves the + * task_group without a sched it's inited on, resolving to %NULL. See + * scx_cgroup_return_subtree(). + * + * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites + * write-lock it, and tg on/offline can't overlap such callers as a css's files + * are created after online and drained before offline. + */ +static struct scx_sched *scx_tg_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup) + tg = &root_task_group; + /* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */ + return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL; +} + +/** + * scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates + * @tg: task_group of interest + * + * Knobs of a cgroup belong to the parent. Deliver the set_* ops to the + * parent task_group's sched, which equals @tg's own sched everywhere except + * at a sub-scheduler attach point, where the sub's parent sched receives + * them. + * + * Return %NULL if the parent task_group has no sched. That can happen when the + * parent's ops.cgroup_init() fails while a sub-scheduler is being disabled. + * + * The callers sit in @tg's cgroup file writes holding the ops rwsem read + * side. That extends scx_tg_sched()'s file-write argument to the parent's + * sched read: a parent css outlives its children's files. + */ +static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup || !tg->css.parent) + return scx_tg_sched(&root_task_group); + return scx_tg_sched(css_tg(tg->css.parent)); +} + int scx_tg_online(struct task_group *tg) { - struct scx_sched *sch = scx_root; int ret = 0; WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); if (scx_cgroup_enabled) { + struct scx_sched *sch; + + /* + * The cgroup lifetime notifier populates cgrp->scx_sched before + * css_online, but only on the default hierarchy. Sub-scheds are + * attached to the cgroup2 hierarchy, so a cgroup1 task_group + * always belongs to the root sched. + */ + if (cgroup_on_dfl(tg->css.cgroup)) + sch = scx_cgroup_sched(tg->css.cgroup); + else + sch = scx_tg_sched(&root_task_group); + if (SCX_HAS_OP(sch, cgroup_init)) { struct scx_cgroup_init_args args = { .weight = tg->scx.weight, @@ -4409,10 +4769,12 @@ int scx_tg_online(struct task_group *tg) ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); if (ret) - ret = ops_sanitize_err(sch, "cgroup_init", ret); + ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); } - if (ret == 0) + if (ret == 0) { + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; + } } else { tg->scx.flags |= SCX_TG_ONLINE; } @@ -4422,19 +4784,30 @@ int scx_tg_online(struct task_group *tg) void scx_tg_offline(struct task_group *tg) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = tg->scx.sched; WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && - (tg->scx.flags & SCX_TG_INITED)) + /* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */ + if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) && + SCX_HAS_OP(sch, cgroup_exit)) SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); + tg->scx.sched = NULL; tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); } +/* + * @p's sched for the cgroup migration paths. Stable as re-homes happen either + * at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(), + * both while holding cgroup_mutex. + */ +static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p) +{ + return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex)); +} + int scx_cgroup_can_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; int ret; @@ -4443,6 +4816,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) return 0; cgroup_taskset_for_each(p, css, tset) { + struct scx_sched *sch = scx_cgroup_task_sched(p); struct cgroup *from = tg_cgrp(task_group(p)); struct cgroup *to = tg_cgrp(css_tg(css)); @@ -4456,11 +4830,22 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) if (from == to) continue; + /* + * The cgroup_move ops are delivered to @p's sched, and only for + * moves that don't re-home @p. A re-homing move changes the dfl + * cgroup's sched and is reported through the + * exit_task/init_task pair that the re-homing generates. + */ + if (!sch || sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp)) + continue; + if (SCX_HAS_OP(sch, cgroup_prep_move)) { ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, p, from, css->cgroup); - if (ret) + if (ret) { + ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); goto err; + } } p->scx.cgrp_moving_from = from; @@ -4470,31 +4855,33 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) err: cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; } - return ops_sanitize_err(sch, "cgroup_prep_move", ret); + return ret; } void scx_cgroup_move_task(struct task_struct *p) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; if (!scx_cgroup_enabled) return; /* - * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's - * cgroup changes. Migration keys off css rather than cgroup identity, - * so it can hand an unchanged-cgroup task here with cgrp_moving_from - * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. + * Migration keys off css rather than cgroup identity, so it can hand an + * unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to + * report to the BPF scheduler then, so skip it and keep prep_move and + * move paired. */ - if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) + sch = scx_cgroup_task_sched(p); + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); @@ -4503,7 +4890,6 @@ void scx_cgroup_move_task(struct task_struct *p) void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; @@ -4511,8 +4897,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) return; cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; @@ -4524,9 +4912,9 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); @@ -4540,9 +4928,9 @@ void scx_group_set_idle(struct task_group *tg, bool idle) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ @@ -4557,9 +4945,9 @@ void scx_group_set_bandwidth(struct task_group *tg, struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || tg->scx.bw_quota_us != quota_us || tg->scx.bw_burst_us != burst_us)) @@ -4585,7 +4973,7 @@ static struct cgroup *root_cgroup(void) * for cgroup_mutex deadlocks with cgroup teardown, which holds it while * draining a set_* file write blocked on the rwsem behind the writer. */ -static void scx_cgroup_lock(void) +void scx_cgroup_lock(void) { cgroup_lock(); #ifdef CONFIG_EXT_GROUP_SCHED @@ -4593,7 +4981,7 @@ static void scx_cgroup_lock(void) #endif } -static void scx_cgroup_unlock(void) +void scx_cgroup_unlock(void) { #ifdef CONFIG_EXT_GROUP_SCHED percpu_up_write(&scx_cgroup_ops_rwsem); @@ -4606,26 +4994,6 @@ static inline void scx_cgroup_lock(void) {} static inline void scx_cgroup_unlock(void) {} #endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -#ifdef CONFIG_EXT_SUB_SCHED -static struct cgroup *sch_cgroup(struct scx_sched *sch) -{ - return sch->cgrp; -} - -/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) -{ - struct cgroup *pos; - struct cgroup_subsys_state *css; - - cgroup_for_each_live_descendant_pre(pos, css, cgrp) - rcu_assign_pointer(pos->scx_sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /* * Omitted operations: * @@ -4669,8 +5037,7 @@ DEFINE_SCHED_CLASS(ext) = { #endif }; -static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, - struct scx_sched *sch) +s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) { s32 cpu; @@ -4760,7 +5127,7 @@ static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) goto out_unlock_dsq; /* - * Mark dead by invalidating ->id to prevent dispatch_enqueue() from + * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks. @@ -4780,8 +5147,6 @@ static void scx_cgroup_exit(struct scx_sched *sch) { struct cgroup_subsys_state *css; - scx_cgroup_enabled = false; - /* * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk * cgroups and exit all the inited ones, all online cgroups are exited. @@ -4789,14 +5154,13 @@ static void scx_cgroup_exit(struct scx_sched *sch) css_for_each_descendant_post(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - if (!(tg->scx.flags & SCX_TG_INITED)) - continue; - tg->scx.flags &= ~SCX_TG_INITED; - - if (!sch->ops.cgroup_exit) - continue; - - SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + /* also clear the sched of tgs whose ops.cgroup_init() failed */ + tg->scx.sched = NULL; + if (tg->scx.flags & SCX_TG_INITED) { + tg->scx.flags &= ~SCX_TG_INITED; + if (sch->ops.cgroup_exit) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + } } } @@ -4811,34 +5175,29 @@ static int scx_cgroup_init(struct scx_sched *sch) */ css_for_each_descendant_pre(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - struct scx_cgroup_init_args args = { - .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us, - }; - if ((tg->scx.flags & - (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) + if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) continue; - if (!sch->ops.cgroup_init) { - tg->scx.flags |= SCX_TG_INITED; - continue; + if (sch->ops.cgroup_init) { + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + return ret; + } } - ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, - css->cgroup, &args); - if (ret) { - scx_error(sch, "ops.cgroup_init() failed (%d)", ret); - return ret; - } + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_INITED; } - WARN_ON_ONCE(scx_cgroup_enabled); - scx_cgroup_enabled = true; - return 0; } @@ -4908,8 +5267,10 @@ static const struct attribute_group scx_global_attr_group = { static void free_pnode(struct scx_sched_pnode *pnode); static void free_exit_info(struct scx_exit_info *ei); +static const char *scx_exit_reason(enum scx_exit_kind kind); +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); @@ -4959,11 +5320,13 @@ static void scx_sched_free_rcu_work(struct work_struct *work) struct scx_dispatch_q *dsq; int cpu, node; + irq_work_sync(&sch->propagate_exit_irq_work); irq_work_sync(&sch->disable_irq_work); kthread_destroy_worker(sch->helper); timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #ifdef CONFIG_EXT_SUB_SCHED kfree(sch->cgrp_path); @@ -4985,7 +5348,21 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - exit_dsq(bypass_dsq(sch, cpu)); + /* remove the queued ecaps sync so the pcpu can be freed */ + scx_discard_ecaps_to_sync(cpu, pcpu); + + /* + * Bypass blocks new kicks. Flush the kick irq_work so this + * pcpu's to_kick_node is off the list before it is freed. + */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); + WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); + free_cpumask_var(pcpu->cpus_to_kick); + free_cpumask_var(pcpu->cpus_to_kick_if_idle); + free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_wait); + + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); @@ -4994,6 +5371,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_pnode(sch->pnode[node]); kfree(sch->pnode); + scx_free_pshards(sch); + rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); do { rhashtable_walk_start(&rht_iter); @@ -5011,6 +5390,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work) scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); + + /* @sch is completely inactive by now */ + scx_dec_has_subs(sch); + kfree(sch); } @@ -5043,26 +5426,62 @@ static ssize_t scx_attr_events_show(struct kobject *kobj, int at = 0; scx_read_events(sch, &events); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) (at += scx_attr_event_show(buf, at, &events, name)) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT return at; } SCX_ATTR(events); +#ifdef CONFIG_EXT_SUB_SCHED +static const char *scx_cap_names[__SCX_NR_CAPS] = { + [__SCX_CAP_ENQ_IMMED] = "enq_immed", + [__SCX_CAP_ENQ] = "enq", + [__SCX_CAP_PREEMPT] = "preempt", + [__SCX_CAP_PERF] = "perf", +}; + +static ssize_t scx_attr_caps_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + u32 npossible = num_possible_cpus(); + struct scx_cmask *agg __free(kfree) = + kzalloc(struct_size(agg, bits, SCX_CMASK_NR_WORDS(npossible)), GFP_KERNEL); + unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL); + ssize_t count = 0; + s32 cap, si; + + if (!agg || !agg_bm) + return -ENOMEM; + + for (cap = 0; cap < __SCX_NR_CAPS; cap++) { + SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS); + + scx_cmask_init(agg, 0, npossible); + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask; + + scx_cmask_reframe(snap, cm->base, cm->nr_cids); + scx_cmask_copy(snap, cm); + scx_cmask_or(agg, snap); + } + /* %*pbl takes unsigned long bitmap layout, convert from u64 */ + bitmap_from_arr64(agg_bm, agg->bits, npossible); + count += sysfs_emit_at(buf, count, "%s: %*pbl\n", + scx_cap_names[cap], npossible, agg_bm); + } + return count; +} +SCX_ATTR(caps); +#endif /* CONFIG_EXT_SUB_SCHED */ + static struct attribute *scx_sched_attrs[] = { &scx_attr_ops.attr, &scx_attr_events.attr, +#ifdef CONFIG_EXT_SUB_SCHED + &scx_attr_caps.attr, +#endif NULL, }; ATTRIBUTE_GROUPS(scx_sched); @@ -5149,6 +5568,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) /** * handle_lockup - sched_ext common lockup handler + * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext @@ -5158,7 +5578,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) { struct scx_sched *sch; va_list args; @@ -5174,7 +5594,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); - ret = scx_verror(sch, fmt, args); + ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); va_end(args); return ret; default: @@ -5184,6 +5604,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) /** * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler + * @stalled_mask: bit mask of stalled CPUs * * While there are various reasons why RCU CPU stalls can occur on a system * that may not be caused by the current BPF scheduler, try kicking out the @@ -5194,9 +5615,46 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) * resolve the reported RCU stall. %false if sched_ext is not enabled or someone * else already initiated abort. */ -bool scx_rcu_cpu_stall(void) +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { - return handle_lockup("RCU CPU stall detected!"); + struct scx_sched *sch; + struct scx_exit_info *ei; + int exit_cpu; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return false; + + switch (scx_enable_state()) { + case SCX_ENABLING: + case SCX_ENABLED: + break; + default: + return false; + } + + exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask); + ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, SCX_EXIT_ERROR)) + return false; + +#ifdef CONFIG_STACKTRACE + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)", + cpumask_pr_args(stalled_mask)); + ei->kind = SCX_EXIT_ERROR; + ei->reason = scx_exit_reason(SCX_EXIT_ERROR); + ei->exit_cpu = exit_cpu; + cpumask_copy(sch->stall_cpus, stalled_mask); + + irq_work_queue(&sch->disable_irq_work); + return true; } /** @@ -5211,53 +5669,39 @@ bool scx_rcu_cpu_stall(void) */ void scx_softlockup(u32 dur_s) { - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + int cpu = smp_processor_id(); + + if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); + cpu, dur_s); } -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); -} - -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - /** * scx_hardlockup - sched_ext hardlockup handler + * @cpu: the target CPU * * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting * numerous affinitized tasks in a single queue and directing all CPUs at it. * Try kicking out the current scheduler in an attempt to recover the system to * a good state before taking more drastic actions. * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). + * Called from NMI. Aborting the scheduler sets ->aborting throughout the + * hierarchy before returning, which is what breaks the dispatch-path live-locks + * that can hard-lock CPUs. * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. */ bool scx_hardlockup(int cpu) { - if (!rcu_access_pointer(scx_root)) + if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu)) return false; - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); return true; } @@ -5266,7 +5710,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, u32 nr_donor_target, u32 nr_donee_target) { struct rq *donor_rq = cpu_rq(donor); - struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); + struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor); struct task_struct *p, *n; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; @@ -5314,7 +5758,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (donee >= nr_cpu_ids) continue; - donee_dsq = bypass_dsq(sch, donee); + donee_dsq = scx_bypass_dsq(sch, donee); /* * $p's rq is not locked but $p's DSQ lock protects its @@ -5335,7 +5779,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); - dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); /* * $donee might have been idle and need to be woken up. No need @@ -5350,6 +5794,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { list_move_tail(&cursor.node, &n->scx.dsq_list.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); cpu_relax(); raw_spin_rq_lock_irq(donor_rq); @@ -5360,6 +5805,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, list_del_init(&cursor.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); return nr_balanced; @@ -5378,7 +5824,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* count the target tasks and CPUs */ for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); nr_tasks += nr; nr_cpus++; @@ -5400,7 +5846,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) cpumask_set_cpu(cpu, donee_mask); } @@ -5411,7 +5857,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) break; if (cpumask_test_cpu(cpu, donee_mask)) continue; - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) continue; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, @@ -5422,7 +5868,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) resched_cpu(cpu); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); after_min = min(nr, after_min); after_max = max(nr, after_max); @@ -5448,7 +5894,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer) int node; u32 intv_us; - if (!bypass_dsp_enabled(sch)) + if (!scx_bypass_dsp_enabled(sch)) return; for_each_node_with_cpus(node) @@ -5514,9 +5960,9 @@ static void enable_bypass_dsp(struct scx_sched *sch) * dispatch enabled while a descendant is bypassing, which is all that's * required. * - * bypass_dsp_enabled() test is used to determine whether to enter the - * bypass dispatch handling path from both bypassing and hosting scheds. - * Bump enable depth on both @sch and bypass dispatch host. + * scx_bypass_dsp_enabled() test is used to determine whether to enter + * the bypass dispatch handling path from both bypassing and hosting + * scheds. Bump enable depth on both @sch and bypass dispatch host. */ ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); WARN_ON_ONCE(ret <= 0); @@ -5536,7 +5982,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) } /* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp(struct scx_sched *sch) +void scx_disable_bypass_dsp(struct scx_sched *sch) { s32 ret; @@ -5552,6 +5998,38 @@ static void disable_bypass_dsp(struct scx_sched *sch) } } +/** + * unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass + * @rq: rq of the cpu leaving bypass + * @pos: scheduler that just left bypass on @rq's cpu + * @pcpu: @pos's per-cpu state for @rq's cpu + * + * A sched leaving bypass is owed the ops.update_idle() calls suppressed while + * bypassing. A cpu that goes idle during the bypass window and stays idle won't + * produce a notification. Arm a re-notify that scx_bypass()'s resched flushes + * on the next idle pick. + * + * An acute case is ops.sub_attach(). If the parent grants the child cids while + * attaching, when attach is complete and bypass is lifted, the child may hold + * idle cids it never saw go idle. + * + * The root is no exception as bypass suppresses its notifications the same way. + * However, the root uses a separate per-rq flag so its re-notify keeps working + * even when !CONFIG_EXT_SUB_SCHED. + */ +static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, + struct scx_sched_pcpu *pcpu) +{ + if (!pos->level) { + rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY; + return; + } +#ifdef CONFIG_EXT_SUB_SCHED + pcpu->idle_renotify = true; + rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; +#endif +} + /** * scx_bypass - [Un]bypass scx_ops and guarantee forward progress * @sch: sched to bypass @@ -5573,18 +6051,18 @@ static void disable_bypass_dsp(struct scx_sched *sch) * * - ops.dispatch() is ignored. * - * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice + * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue with core_sched_at touched. + * the tail of the queue. * * - pick_next_task() suppresses zero slice warning. * * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM * operations. * - * - scx_prio_less() reverts to the default core_sched_at order. + * - scx_prio_less() reverts to the default runnable_at order. */ -static void scx_bypass(struct scx_sched *sch, bool bypass) +void scx_bypass(struct scx_sched *sch, bool bypass) { struct scx_sched *pos; unsigned long flags; @@ -5635,11 +6113,17 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) scx_for_each_descendant_pre(pos, sch) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING; - if (pos->bypass_depth) + if (pos->bypass_depth) { pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; - else + } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; + if (was_bypassing) { + unbypass_renotify_idle(rq, pos, pcpu); + scx_unbypass_replay_ecaps(rq, pos); + } + } } raw_spin_unlock(&scx_sched_lock); @@ -5650,6 +6134,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) * sees scx_bypassing() before moving tasks to SCX. */ if (!scx_enabled()) { + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); continue; } @@ -5666,6 +6151,13 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; + /* + * Bypass trumps protection. Cycling clears for queued + * tasks but current task needs explicit stripping. + */ + if (bypass && task_current(rq, p)) + scx_task_slice_ended(rq, p); + /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; @@ -5676,12 +6168,13 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) if (cpu_online(cpu) || cpu == smp_processor_id()) resched_curr(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } /* disarming must come after moving all tasks out of the bypass DSQs */ if (!bypass) - disable_bypass_dsp(sch); + scx_disable_bypass_dsp(sch); unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } @@ -5728,12 +6221,18 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "disabled by sysrq-S"; case SCX_EXIT_PARENT: return "parent exiting"; + case SCX_EXIT_PARENT_KILL: + return "killed by parent scheduler"; case SCX_EXIT_ERROR: return "runtime error"; case SCX_EXIT_ERROR_BPF: return "scx_bpf_error"; case SCX_EXIT_ERROR_STALL: return "runnable task stall"; + case SCX_EXIT_ERROR_REENQ: + return "reenqueue limit"; + case SCX_EXIT_ERROR_RESCUE: + return "rescue bandwidth overload"; default: return ""; } @@ -5747,6 +6246,8 @@ static void free_kick_syncs(void) struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *to_free; + /* flush the pending kick before freeing @ksyncs */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); to_free = rcu_replace_pointer(*ksyncs, NULL, true); if (to_free) kvfree_rcu(to_free, rcu); @@ -5773,63 +6274,70 @@ static void refresh_watchdog(void) cancel_delayed_work_sync(&scx_watchdog_work); } -static s32 scx_link_sched(struct scx_sched *sch) +s32 scx_link_sched(struct scx_sched *sch) { - const char *err_msg = ""; - s32 ret = 0; - - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { + scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock) /* for the parent bypass check */ + scoped_guard(raw_spinlock, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *parent = scx_parent(sch); if (parent) { + s32 ret; + /* - * scx_claim_exit() propagates exit_kind transition to - * its sub-scheds while holding scx_sched_lock - either - * we can see the parent's non-NONE exit_kind or the - * parent can shoot us down. + * Bypass state is spread across per-cpu flags and a + * depth count, so inheriting it is tricky and has no + * valid use case. Refuse it. */ - if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { - err_msg = "parent disabled"; - ret = -ENOENT; - break; + if (READ_ONCE(parent->bypass_depth)) { + scx_error(sch, "parent bypassing (%d)", -EBUSY); + return -EBUSY; } ret = rhashtable_lookup_insert_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); if (ret) { - err_msg = "failed to insert into scx_sched_hash"; - break; + scx_error(sch, "failed to insert into scx_sched_hash (%d)", + ret); + return ret; } - list_add_tail(&sch->sibling, &parent->children); + list_add_tail_rcu(&sch->sibling, &parent->children); + + /* + * Pairs with the mb after the ->aborting assertion in + * scx_claim_exit(). Either we see ->aborting and back + * out, or the exit path sees us and exits us. + */ + smp_mb(); + if (unlikely(READ_ONCE(parent->aborting))) { + rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, + scx_sched_hash_params); + list_del_rcu(&sch->sibling); + scx_error(sch, "parent disabled (%d)", -ENOENT); + return -ENOENT; + } + + sch->linked = true; } #endif /* CONFIG_EXT_SUB_SCHED */ list_add_tail_rcu(&sch->all, &scx_sched_all); } - /* - * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after - * the guard above is released. - */ - if (ret) { - scx_error(sch, "%s (%d)", err_msg, ret); - return ret; - } - refresh_watchdog(); return 0; } -static void scx_unlink_sched(struct scx_sched *sch) +void scx_unlink_sched(struct scx_sched *sch) { scoped_guard(raw_spinlock_irq, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED - if (scx_parent(sch)) { + if (sch->linked) { rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); - list_del_init(&sch->sibling); + list_del_rcu(&sch->sibling); + sch->linked = false; } #endif /* CONFIG_EXT_SUB_SCHED */ list_del_rcu(&sch->all); @@ -5843,13 +6351,13 @@ static void scx_unlink_sched(struct scx_sched *sch) * @sch. Once @sch becomes empty during disable, there's no point in dumping it. * This prevents calling dump ops on a dead sch. */ -static void scx_disable_dump(struct scx_sched *sch) +void scx_disable_dump(struct scx_sched *sch) { guard(raw_spinlock_irqsave)(&scx_dump_lock); sch->dump_disabled = true; } -static void scx_log_sched_disable(struct scx_sched *sch) +void scx_log_sched_disable(struct scx_sched *sch) { struct scx_exit_info *ei = sch->exit_info; const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; @@ -5869,219 +6377,6 @@ static void scx_log_sched_disable(struct scx_sched *sch) } } -#ifdef CONFIG_EXT_SUB_SCHED -static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); - -static void drain_descendants(struct scx_sched *sch) -{ - /* - * Child scheds that finished the critical part of disabling will take - * themselves off @sch->children. Wait for it to drain. As propagation - * is recursive, empty @sch->children means that all proper descendant - * scheds reached unlinking stage. - */ - wait_event(scx_unlink_waitq, list_empty(&sch->children)); -} - -static void scx_fail_parent(struct scx_sched *sch, - struct task_struct *failed, s32 fail_code) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - - scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", - fail_code, failed->comm, failed->pid); - - /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. - */ - scx_bypass(parent, true); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (scx_task_on_sched(parent, p)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } - } - scx_task_iter_stop(&sti); -} - -static void scx_sub_disable(struct scx_sched *sch) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - int ret; - - /* - * Guarantee forward progress and wait for descendants to be disabled. - * To limit disruptions, $parent is not bypassed. Tasks are fully - * prepped and then inserted back into $parent. - */ - scx_bypass(sch, true); - drain_descendants(sch); - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * An enable that failed before scx_link_sched() never owned a cgroup or - * task and won't be waited on by an ancestor's drain_descendants(). - * Nothing to reparent and walking the tasks can misbehave as the task - * ownership invariant (either owned by self or parent) does not hold. - */ - if (list_empty(&sch->sibling)) - goto dump; - - set_cgroup_sched(sch_cgroup(sch), parent); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* filter out duplicate visits */ - if (scx_task_on_sched(parent, p)) - continue; - - /* - * By the time control reaches here, all linked descendant - * schedulers should have been disabled. - */ - WARN_ON_ONCE(!scx_task_on_sched(sch, p)); - - /* - * @p is pinned by the iter: css_task_iter_next() takes a - * reference and holds it until the next iter_next() call, so - * @p->usage is guaranteed > 0. - */ - get_task_struct(p); - - scx_task_iter_unlock(&sti); - - /* - * $p is READY or ENABLED on @sch. Initialize for $parent, - * disable and exit from @sch, and then switch over to $parent. - * - * If a task fails to initialize for $parent, the only available - * action is disabling $parent too. While this allows disabling - * of a child sched to cause the parent scheduler to fail, the - * failure can only originate from ops.init_task() of the - * parent. A child can't directly affect the parent through its - * own failures. - */ - ret = __scx_init_task(parent, p, false); - if (ret) { - scx_fail_parent(sch, p, ret); - put_task_struct(p); - break; - } - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on @sch (the - * sched @p was on at that point), not on $parent. - * $parent's just-completed init is owed an exit_task() - * and we issue it here. - */ - scx_sub_init_cancel_task(parent, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent and override the state to READY to account - * for $p having already been initialized. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - - /* - * A task on a non-ext class, possible under an - * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is - * enabled by switching_to_scx() if it switches over. - */ - if (p->sched_class == &ext_sched_class) - scx_enable_task(parent, p); - } - - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - } - scx_task_iter_stop(&sti); - -dump: - scx_disable_dump(sch); - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are moved off of @sch but there may still be on-going - * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use - * the expedited version as ancestors may be waiting in bypass mode. - * Also, tell the parent that there is no need to keep running bypass - * DSQs for us. - */ - synchronize_rcu_expedited(); - disable_bypass_dsp(sch); - - scx_unlink_sched(sch); - - mutex_unlock(&scx_enable_mutex); - - /* - * @sch is now unlinked from the parent's children list. Notify and call - * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called - * after unlinking and releasing all locks. See scx_claim_exit(). - */ - wake_up_all(&scx_unlink_waitq); - - if (parent->ops.sub_detach && sch->sub_attached) { - struct scx_sub_detach_args sub_detach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - SCX_CALL_OP(parent, sub_detach, NULL, - &sub_detach_args); - } - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline void drain_descendants(struct scx_sched *sch) { } -static inline void scx_sub_disable(struct scx_sched *sch) { } -#endif /* CONFIG_EXT_SUB_SCHED */ - static void scx_root_disable(struct scx_sched *sch) { struct scx_task_iter sti; @@ -6119,10 +6414,11 @@ static void scx_root_disable(struct scx_sched *sch) WRITE_ONCE(scx_switching_all, false); /* - * Shut down cgroup support before tasks so that the cgroup attach path - * doesn't race against scx_disable_and_exit_task(). + * Shut down cgroup support before tasks so that the cgroup attach and + * migration paths don't race against scx_disable_and_exit_task(). */ scx_cgroup_lock(); + scx_cgroup_enabled = false; scx_cgroup_exit(sch); scx_cgroup_unlock(); @@ -6162,12 +6458,9 @@ static void scx_root_disable(struct scx_sched *sch) percpu_up_write(&scx_fork_rwsem); /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. + * Re-balance the dl_server bandwidth reservations: detach ext_server + * (no more sched_ext tasks) and reinstate fair_server if it was + * previously detached because we were running in full mode. * * Unlike the enable path, this runs on a recovery path that cannot * fail, so we use dl_server_swap_bw() to atomically free ext_server's @@ -6180,8 +6473,6 @@ static void scx_root_disable(struct scx_sched *sch) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); if (was_switched_all) { @@ -6210,14 +6501,23 @@ static void scx_root_disable(struct scx_sched *sch) if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + /* + * @sch's non-ops programs such as timers and tracers can fire after + * ops.exit(). Now that exit is complete, stop scx_prog_sched() from + * resolving to @sch and drain in-flight resolvers. + */ + WRITE_ONCE(sch->dead, true); + synchronize_rcu(); + scx_unlink_sched(sch); /* - * scx_root clearing must be inside cpus_read_lock(). See - * handle_hotplug(). + * scx_root clearing and cid table retirement must be inside + * cpus_read_lock(). See handle_hotplug(). */ cpus_read_lock(); RCU_INIT_POINTER(scx_root, NULL); + scx_cid_retire_tables(); cpus_read_unlock(); /* @@ -6229,7 +6529,9 @@ static void scx_root_disable(struct scx_sched *sch) if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj); #endif - kobject_del(&sch->kobj); + /* not added if enable failed before scx_sched_sysfs_add() */ + if (sch->kobj.state_in_sysfs) + kobject_del(&sch->kobj); free_kick_syncs(); @@ -6240,12 +6542,36 @@ static void scx_root_disable(struct scx_sched *sch) scx_bypass(sch, false); } +/** + * scx_propagate_exit_irq_workfn - Claim SCX_EXIT_PARENT on the exiting subtree + * @irq_work: &scx_sched.propagate_exit_irq_work + * + * Queued by scx_claim_exit() after a non-PARENT claim. Claims SCX_EXIT_PARENT + * on each descendant, giving every one its own disable work - most of disabling + * is serialized but ops.exit() can take arbitrarily long and running them in + * separate helper kthreads parallelizes it. No recursion as only non-PARENT + * claims propagate. + */ +static void scx_propagate_exit_irq_workfn(struct irq_work *irq_work) +{ + struct scx_sched *sch = container_of(irq_work, struct scx_sched, + propagate_exit_irq_work); + struct scx_sched *pos; + + scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { + scx_for_each_descendant_pre(pos, sch) + scx_disable(pos, SCX_EXIT_PARENT); + } +} + /* * Claim the exit on @sch. The caller must ensure that the helper kthread work * is kicked before the current task can be preempted. Once exit_kind is * claimed, scx_error() can no longer trigger, so if the current task gets * preempted and the BPF scheduler fails to schedule it back, the helper work * will never be kicked and the whole system can wedge. + * + * Lock-free and safe to call from any context including NMI. */ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) { @@ -6259,35 +6585,31 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) return false; - /* - * Some CPUs may be trapped in the dispatch paths. Set the aborting - * flag to break potential live-lock scenarios, ensuring we can - * successfully reach scx_bypass(). - */ - WRITE_ONCE(sch->aborting, true); + if (kind == SCX_EXIT_PARENT) { + /* an ancestor is already sweeping the subtree */ + WRITE_ONCE(sch->aborting, true); + } else { + struct scx_sched *pos; - /* - * Propagate exits to descendants immediately. Each has a dedicated - * helper kthread and can run in parallel. While most of disabling is - * serialized, running them in separate threads allows parallelizing - * ops.exit(), which can take arbitrarily long prolonging bypass mode. - * - * To guarantee forward progress, this propagation must be in-line so - * that ->aborting is synchronously asserted for all sub-scheds. The - * propagation is also the interlocking point against sub-sched - * attachment. See scx_link_sched(). - * - * This doesn't cause recursions as propagation only takes place for - * non-propagation exits. - */ - if (kind != SCX_EXIT_PARENT) { - scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { - struct scx_sched *pos; + /* + * CPUs may be live-locked in the dispatch paths of @sch or its + * descendants, which ->aborting breaks. Sweep the subtree + * locklessly so that this works from NMI. smp_store_mb() orders + * each node's ->aborting store before its children are walked - + * either we see a racing scx_link_sched() on ->children or it + * sees ->aborting. + */ + scoped_guard (rcu) { scx_for_each_descendant_pre(pos, sch) - scx_disable(pos, SCX_EXIT_PARENT); + smp_store_mb(pos->aborting, true); } + + irq_work_queue(&sch->propagate_exit_irq_work); } + /* fired after ->aborting is set so callbacks can't delay recovery */ + trace_sched_ext_exit(sch, kind); + return true; } @@ -6329,7 +6651,7 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) * as a noop. Syncing the irq_work first is required to guarantee the * kthread work has been queued before waiting for it. */ -static void scx_flush_disable_work(struct scx_sched *sch) +void scx_flush_disable_work(struct scx_sched *sch) { int kind; @@ -6349,7 +6671,7 @@ static void dump_newline(struct seq_buf *s) seq_buf_putc(s, '\n'); } -static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) +__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...) { va_list args; @@ -6381,7 +6703,7 @@ static void dump_stack_trace(struct seq_buf *s, const char *prefix, unsigned int i; for (i = 0; i < len; i++) - dump_line(s, "%s%pS", prefix, (void *)bt[i]); + scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]); } static void ops_dump_init(struct seq_buf *s, const char *prefix) @@ -6431,7 +6753,7 @@ static void ops_dump_flush(void) */ c = *end; *end = '\0'; - dump_line(dd->s, "%s%s", dd->prefix, line); + scx_dump_line(dd->s, "%s%s", dd->prefix, line); if (c == '\0') break; @@ -6475,21 +6797,19 @@ static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_d (unsigned long long)p->scx.dsq->id); dump_newline(s); - dump_line(s, " %c%c %s[%d] %s%s %+ldms", - marker, task_state_to_char(p), p->comm, p->pid, - own_marker, sch_id_buf, - jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); - dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", - scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, - p->scx.flags & ~SCX_TASK_STATE_MASK, - p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, - ops_state >> SCX_OPSS_QSEQ_SHIFT); - dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", - p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); - dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", - p->scx.dsq_vtime, p->scx.slice, p->scx.weight); - dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), - p->migration_disabled); + scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms", + marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf, + jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); + scx_dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", + scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, + p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, + ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); + scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", + p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); + scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", + p->scx.dsq_vtime, p->scx.slice, p->scx.weight); + scx_dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), + p->migration_disabled); if (SCX_HAS_OP(sch, dump_task)) { ops_dump_init(s, " "); @@ -6511,6 +6831,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, bool dump_all_tasks) { struct rq *rq = cpu_rq(cpu); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); struct rq_flags rf; struct task_struct *p; struct seq_buf ns; @@ -6536,28 +6857,27 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", - cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); - dump_line(&ns, " curr=%s[%d] class=%ps", - rq->curr->comm, rq->curr->pid, - rq->curr->sched_class); - if (!cpumask_empty(rq->scx.cpus_to_kick)) - dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick)); - if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) - dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); - if (!cpumask_empty(rq->scx.cpus_to_preempt)) - dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(rq->scx.cpus_to_preempt)); - if (!cpumask_empty(rq->scx.cpus_to_wait)) - dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(rq->scx.cpus_to_wait)); + scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, + rq->scx.ops_qseq, rq->scx.kick_sync); + scx_rescue_dump(&ns, rq); + scx_dump_line(&ns, " curr=%s[%d] class=%ps", + rq->curr->comm, rq->curr->pid, rq->curr->sched_class); + if (!cpumask_empty(pcpu->cpus_to_kick)) + scx_dump_line(&ns, " cpus_to_kick : %*pb", + cpumask_pr_args(pcpu->cpus_to_kick)); + if (!cpumask_empty(pcpu->cpus_to_kick_if_idle)) + scx_dump_line(&ns, " idle_to_kick : %*pb", + cpumask_pr_args(pcpu->cpus_to_kick_if_idle)); + if (!cpumask_empty(pcpu->cpus_to_preempt)) + scx_dump_line(&ns, " cpus_to_preempt: %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_wait)) + scx_dump_line(&ns, " cpus_to_wait : %*pb", + cpumask_pr_args(pcpu->cpus_to_wait)); if (!cpumask_empty(rq->scx.cpus_to_sync)) - dump_line(&ns, " cpus_to_sync : %*pb", - cpumask_pr_args(rq->scx.cpus_to_sync)); + scx_dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); used = seq_buf_used(&ns); if (SCX_HAS_OP(sch, dump_cpu)) { @@ -6625,25 +6945,25 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, #ifdef CONFIG_EXT_SUB_SCHED if (sch->level == 0) - dump_line(&s, "%s: root", sch->ops.name); + scx_dump_line(&s, "%s: root", sch->ops.name); else - dump_line(&s, "%s: sub%d-%llu %s", - sch->ops.name, sch->level, sch->ops.sub_cgroup_id, - sch->cgrp_path); + scx_dump_line(&s, "%s: sub%d-%llu %s", + sch->ops.name, sch->level, sch->ops.sub_cgroup_id, + sch->cgrp_path); #endif if (ei->kind == SCX_EXIT_NONE) { - dump_line(&s, "Debug dump triggered by %s", ei->reason); + scx_dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { if (ei->exit_cpu >= 0) - dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", - current->comm, current->pid, ei->kind, - ei->exit_cpu); + scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", + current->comm, current->pid, ei->kind, + ei->exit_cpu); else - dump_line(&s, "%s[%d] triggered exit kind %d:", - current->comm, current->pid, ei->kind); - dump_line(&s, " %s (%s)", ei->reason, ei->msg); + scx_dump_line(&s, "%s[%d] triggered exit kind %d:", + current->comm, current->pid, ei->kind); + scx_dump_line(&s, " %s (%s)", ei->reason, ei->msg); dump_newline(&s); - dump_line(&s, "Backtrace:"); + scx_dump_line(&s, "Backtrace:"); dump_stack_trace(&s, " ", ei->bt, ei->bt_len); } @@ -6654,38 +6974,37 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, } dump_newline(&s); - dump_line(&s, "CPU states"); - dump_line(&s, "----------"); + scx_dump_line(&s, "CPU states"); + scx_dump_line(&s, "----------"); /* - * Dump the exit CPU first so it isn't lost to dump truncation, then - * walk the rest in order, skipping the one already dumped. + * Dump stalled CPUs first so they aren't lost to dump truncation, then + * walk the rest in order. Fall back to exit_cpu if no stall mask set. */ - if (ei->exit_cpu >= 0) - scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); - for_each_possible_cpu(cpu) { - if (cpu != ei->exit_cpu) + if (!cpumask_empty(sch->stall_cpus)) { + for_each_cpu(cpu, sch->stall_cpus) scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (!cpumask_test_cpu(cpu, sch->stall_cpus)) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } + } else { + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } } dump_newline(&s); - dump_line(&s, "Event counters"); - dump_line(&s, "--------------"); + scx_dump_line(&s, "Event counters"); + scx_dump_line(&s, "--------------"); scx_read_events(sch, &events); - scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); - scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); - scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); - scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); - scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); - scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); - scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); - scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); - scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) scx_dump_event(s, &events, name) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) memcpy(ei->dump + dump_len - sizeof(trunc_marker), @@ -6703,6 +7022,32 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work) kthread_queue_work(sch->helper, &sch->disable_work); } +/* finish exit_info and kick the disable work, ei->msg must already be set */ +static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu) +{ + struct scx_exit_info *ei = sch->exit_info; + + ei->exit_code = exit_code; +#ifdef CONFIG_STACKTRACE + /* + * stack_trace_save()'s NMI-safety is arch-dependent and undocumented. + * Skip the backtrace when exiting from NMI. + */ + if (kind >= SCX_EXIT_ERROR && !in_nmi()) + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + /* + * Set ei->kind and ->reason for scx_dump_state(). They'll be set again + * in scx_disable_workfn(). + */ + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); + ei->exit_cpu = exit_cpu; + + irq_work_queue(&sch->disable_irq_work); +} + bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, va_list args) @@ -6714,22 +7059,9 @@ bool scx_vexit(struct scx_sched *sch, if (!scx_claim_exit(sch, kind)) return false; - ei->exit_code = exit_code; -#ifdef CONFIG_STACKTRACE - if (kind >= SCX_EXIT_ERROR) - ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); -#endif vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); - /* - * Set ei->kind and ->reason for scx_dump_state(). They'll be set again - * in scx_disable_workfn(). - */ - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); - ei->exit_cpu = exit_cpu; - - irq_work_queue(&sch->disable_irq_work); + scx_finish_exit(sch, kind, exit_code, exit_cpu); return true; } @@ -6776,7 +7108,7 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) if (!pnode) return NULL; - if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { + if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { kfree(pnode); return NULL; } @@ -6784,31 +7116,13 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) return pnode; } -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - union { - struct sched_ext_ops *ops; - struct sched_ext_ops_cid *ops_cid; - }; - bool is_cid_type; - struct bpf_map *arena_map; /* arena ref to transfer to sch */ - int ret; -}; - /* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */ -static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, - struct cgroup *cgrp, - struct scx_sched *parent) +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent) { struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; @@ -6855,7 +7169,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, } for_each_possible_cpu(cpu) { - ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu; @@ -6865,8 +7179,20 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + node = cpu_to_node(cpu); pcpu->sch = sch; INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); +#ifdef CONFIG_EXT_SUB_SCHED + init_llist_node(&pcpu->ecaps_to_sync_node); +#endif + INIT_LIST_HEAD(&pcpu->to_kick_node); + if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { + ret = -ENOMEM; + goto err_free_pcpu; + } } sch->helper = kthread_run_worker(0, "sched_ext_helper"); @@ -6882,6 +7208,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, level * sizeof(parent->ancestors[0])); sch->ancestors[level] = sch; sch->level = level; + sch->id = atomic64_inc_return(&scx_sched_id_cursor); if (ops->timeout_ms) sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); @@ -6891,6 +7218,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, sch->slice_dfl = SCX_SLICE_DFL; atomic_set(&sch->exit_kind, SCX_EXIT_NONE); sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); + sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn); kthread_init_work(&sch->disable_work, scx_disable_workfn); timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); @@ -6902,6 +7230,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, ret = -ENOMEM; goto err_free_lb_cpumask; } + if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_lb_resched_cpumask; + } /* * Copy ops through the right union view. For cid-form the source is * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ @@ -6951,36 +7283,15 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, * disable. Released in scx_sched_free_rcu_work(). */ kobject_get(&parent->kobj); - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, - &parent->sub_kset->kobj, - "sub-%llu", cgroup_id(cgrp)); - } else { - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - } - - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } - - if (ops->sub_attach) { - sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); - if (!sch->sub_kset) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(-ENOMEM); - } - } -#else /* CONFIG_EXT_SUB_SCHED */ - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); } #endif /* CONFIG_EXT_SUB_SCHED */ + /* + * Init the kobj but don't add to sysfs yet. The enable path calls + * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized. + */ + kobject_init(&sch->kobj, &scx_ktype); + /* * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid @@ -6998,17 +7309,27 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, #ifdef CONFIG_EXT_SUB_SCHED err_free_lb_resched: - free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #endif +err_free_lb_resched_cpumask: + free_cpumask_var(sch->bypass_lb_resched_cpumask); err_free_lb_cpumask: free_cpumask_var(sch->bypass_lb_donee_cpumask); err_stop_helper: kthread_destroy_worker(sch->helper); err_free_pcpu: + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + free_cpumask_var(pcpu->cpus_to_kick); + free_cpumask_var(pcpu->cpus_to_kick_if_idle); + free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_wait); + } for_each_possible_cpu(cpu) { if (cpu == bypass_fail_cpu) break; - exit_dsq(bypass_dsq(sch, cpu)); + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); err_free_pnode: @@ -7028,6 +7349,36 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, return ERR_PTR(ret); } +/* + * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler + * implements ops.sub_attach. Called by the enable workfns once @sch's + * sysfs-visible state is initialized. + */ +int scx_sched_sysfs_add(struct scx_sched *sch) +{ +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *parent = scx_parent(sch); + int ret; + + if (parent) + ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj, + "sub-%llu", cgroup_id(sch_cgroup(sch))); + else + ret = kobject_add(&sch->kobj, NULL, "root"); + if (ret < 0) + return ret; + + if (sch->ops.sub_attach) { + sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); + if (!sch->sub_kset) + return -ENOMEM; + } + return 0; +#else + return kobject_add(&sch->kobj, NULL, "root"); +#endif +} + static int check_hotplug_seq(struct scx_sched *sch, const struct sched_ext_ops *ops) { @@ -7052,7 +7403,7 @@ static int check_hotplug_seq(struct scx_sched *sch, return 0; } -static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) { /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the @@ -7069,7 +7420,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) * enabled it. */ if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && - !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { + !(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) { scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); return -EINVAL; } @@ -7180,6 +7531,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) rq->scx.cpuperf_target = SCX_CPUPERF_ONE; } + scx_discard_stale_ecaps_syncs(); + scx_rescue_set_knobs(sch); + /* * Keep CPUs stable during enable so that the BPF scheduler can track * online CPUs by watching ->on/offline_cpu() after ->init(). @@ -7187,10 +7541,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) cpus_read_lock(); /* - * Build the cid mapping before publishing scx_root. The cid kfuncs - * dereference the cid arrays unconditionally once scx_prog_sched() - * returns non-NULL; the rcu_assign_pointer() below pairs with their - * rcu_dereference() to make the populated arrays visible. + * Build the cid mapping into a private under-construction set. It + * becomes visible to readers only through scx_cid_publish_tables() once + * ops.init_cids() has finalized the layout. */ ret = scx_cid_init(sch); if (ret) { @@ -7212,17 +7565,24 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_idle_enable(ops); - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); + /* + * A cid-form scheduler finalizes its cid layout in ops.init_cids(), + * which may call scx_bpf_cid_override(). Run it before the caps and + * shard state are built so the final layout is in effect. + */ + if (sch->is_cid_type && sch->ops_cid.init_cids) { + ret = SCX_CALL_OP_RET(sch, init_cids, NULL); if (ret) { - ret = ops_sanitize_err(sch, "init", ret); + ret = scx_ops_sanitize_err(sch, "init_cids", ret); cpus_read_unlock(); - scx_error(sch, "ops.init() failed (%d)", ret); + scx_error(sch, "ops.init_cids() failed (%d)", ret); goto err_disable; } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } + /* the cid layout is final, expose it to readers */ + scx_cid_publish_tables(); + ret = scx_arena_pool_init(sch); if (ret) { cpus_read_unlock(); @@ -7235,6 +7595,32 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + ret = scx_alloc_pshards(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + scx_init_root_caps(sch); + + /* the cid caps and shards are live now, so ops.init() can query them */ + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = scx_ops_sanitize_err(sch, "init", ret); + cpus_read_unlock(); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_sched_sysfs_add(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -7248,7 +7634,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) cpus_read_unlock(); - ret = validate_ops(sch, ops); + ret = scx_validate_ops(sch, ops); if (ret) goto err_disable; @@ -7324,6 +7710,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (ret) goto err_disable_unlock_all; + WARN_ON_ONCE(scx_cgroup_enabled); + scx_cgroup_enabled = true; + scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { /* @@ -7347,7 +7736,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_task_iter_unlock(&sti); - ret = __scx_init_task(sch, p, false); + ret = __scx_init_task(sch, p, NULL, false); scx_task_iter_relock(&sti, p); @@ -7414,7 +7803,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) queue_flags |= DEQUEUE_CLASS; scoped_guard (sched_change, p, queue_flags) { - p->scx.slice = READ_ONCE(sch->slice_dfl); + scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); p->sched_class = new_class; } } @@ -7425,6 +7814,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); + ret = -EBUSY; goto err_disable; } @@ -7490,357 +7880,6 @@ static void scx_root_enable_workfn(struct kthread_work *work) cmd->ret = 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/* verify that a scheduler can be attached to @cgrp and return the parent */ -static struct scx_sched *find_parent_sched(struct cgroup *cgrp) -{ - struct scx_sched *parent = cgrp->scx_sched; - struct scx_sched *pos; - - lockdep_assert_held(&scx_sched_lock); - - /* can't attach twice to the same cgroup */ - if (parent->cgrp == cgrp) - return ERR_PTR(-EBUSY); - - /* does $parent allow sub-scheds? */ - if (!parent->ops.sub_attach) - return ERR_PTR(-EOPNOTSUPP); - - /* can't insert between $parent and its exiting children */ - list_for_each_entry(pos, &parent->children, sibling) - if (cgroup_is_descendant(pos->cgrp, cgrp)) - return ERR_PTR(-EBUSY); - - return parent; -} - -static bool assert_task_ready_or_enabled(struct task_struct *p) -{ - u32 state = scx_get_task_state(p); - - switch (state) { - case SCX_TASK_READY: - case SCX_TASK_ENABLED: - return true; - default: - WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", - state, p->comm, p->pid); - return false; - } -} - -static void scx_sub_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp; - struct scx_sched *parent, *sch; - struct scx_task_iter sti; - struct task_struct *p; - s32 i, ret; - - mutex_lock(&scx_enable_mutex); - - if (!scx_enabled()) { - ret = -ENODEV; - goto out_unlock; - } - - /* See scx_root_enable_workfn() for the @ops->priv check. */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto out_unlock; - } - - cgrp = cgroup_get_from_id(ops->sub_cgroup_id); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - raw_spin_lock_irq(&scx_sched_lock); - parent = find_parent_sched(cgrp); - if (IS_ERR(parent)) { - raw_spin_unlock_irq(&scx_sched_lock); - ret = PTR_ERR(parent); - goto out_put_cgrp; - } - kobject_get(&parent->kobj); - raw_spin_unlock_irq(&scx_sched_lock); - - /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(cmd, cgrp, parent); - kobject_put(&parent->kobj); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto out_unlock; - } - - ret = scx_link_sched(sch); - if (ret) - goto err_disable; - - if (sch->level >= SCX_SUB_MAX_DEPTH) { - scx_error(sch, "max nesting depth %d violated", - SCX_SUB_MAX_DEPTH); - goto err_disable; - } - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - - if (validate_ops(sch, ops)) - goto err_disable; - - struct scx_sub_attach_args sub_attach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - - ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, - &sub_attach_args); - if (ret) { - ret = ops_sanitize_err(sch, "sub_attach", ret); - scx_error(sch, "parent rejected (%d)", ret); - goto err_disable; - } - sch->sub_attached = true; - - scx_bypass(sch, true); - - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see - * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. - */ - set_cgroup_sched(sch_cgroup(sch), sch); - if (!(cgrp->self.flags & CSS_ONLINE)) { - scx_error(sch, "cgroup is not online"); - goto err_unlock_and_disable; - } - - /* - * Initialize tasks for the new child $sch without exiting them for - * $parent so that the tasks can always be reverted back to $parent - * sched on child init failure. - */ - WARN_ON_ONCE(scx_enabling_sub_sched); - scx_enabling_sub_sched = sch; - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* - * Task iteration may visit the same task twice when racing - * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which - * finished __scx_init_task() and skip if set. - * - * A task may exit and get freed between __scx_init_task() - * completion and scx_enable_task(). In such cases, - * scx_disable_and_exit_task() must exit the task for both the - * parent and child scheds. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) - continue; - - /* @p is pinned by the iter; see scx_sub_disable() */ - get_task_struct(p); - - if (!assert_task_ready_or_enabled(p)) { - ret = -EINVAL; - goto abort; - } - - scx_task_iter_unlock(&sti); - - /* - * As $p is still on $parent, it can't be transitioned to INIT. - * Let's worry about task state later. Use __scx_init_task(). - */ - ret = __scx_init_task(sch, p, false); - if (ret) - goto abort; - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on $parent (the - * sched @p was on at that point), not on @sch. @sch's - * just-completed init is owed an exit_task() and we - * issue it here. - */ - scx_sub_init_cancel_task(sch, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - p->scx.flags |= SCX_TASK_SUB_INIT; - task_rq_unlock(rq, p, &rf); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - /* - * All tasks are prepped. Disable/exit tasks for $parent and enable for - * the new @sch. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * Use clearing of %SCX_TASK_SUB_INIT to detect and skip - * duplicate iterations. - */ - if (!(p->scx.flags & SCX_TASK_SUB_INIT)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p must be either READY or ENABLED. If ENABLED, - * __scx_disabled_and_exit_task() first disables and - * makes it READY. However, after exiting $p, it will - * leave $p as READY. - */ - assert_task_ready_or_enabled(p); - __scx_disable_and_exit_task(parent, p); - - /* - * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and, if it's on - * the ext class, enable. A non-ext task, possible under - * an %SCX_OPS_SWITCH_PARTIAL root, stays READY and is - * enabled by switching_to_scx() if it switches over. - */ - scx_set_task_sched(p, sch); - if (p->sched_class == &ext_sched_class) - scx_enable_task(sch, p); - - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - - scx_enabling_sub_sched = NULL; - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); - kobject_uevent(&sch->kobj, KOBJ_ADD); - ret = 0; - goto out_unlock; - -out_put_cgrp: - cgroup_put(cgrp); -out_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -abort: - put_task_struct(p); - scx_task_iter_stop(&sti); - - /* - * Undo __scx_init_task() for tasks we marked. scx_enable_task() never - * ran for @sch on them, so calling scx_disable_task() here would invoke - * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched - * must stay set until SUB_INIT is cleared from every marked task - - * scx_disable_and_exit_task() reads it when a task exits concurrently. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (p->scx.flags & SCX_TASK_SUB_INIT) { - scx_sub_init_cancel_task(sch, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - scx_enabling_sub_sched = NULL; -err_unlock_and_disable: - /* we'll soon enter disable path, keep bypass on */ - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); -err_disable: - mutex_unlock(&scx_enable_mutex); - /* - * Some enable failures only return an errno (e.g. -ENOMEM from an - * allocation) without calling scx_error(). Record it so - * scx_flush_disable_work() runs the disable and ops.exit() fires. - */ - scx_error(sch, "scx_sub_enable() failed (%d)", ret); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cgroup *cgrp = data; - struct cgroup *parent = cgroup_parent(cgrp); - - if (!cgroup_on_dfl(cgrp)) - return NOTIFY_OK; - - switch (action) { - case CGROUP_LIFETIME_ONLINE: - /* inherit ->scx_sched from $parent */ - if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); - break; - case CGROUP_LIFETIME_OFFLINE: - /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, - SCX_ECODE_RSN_CGROUP_OFFLINE, - "cgroup %llu going offline", cgroup_id(cgrp)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_cgroup_lifetime_nb = { - .notifier_call = scx_cgroup_lifetime_notify, -}; - -static s32 __init scx_cgroup_lifetime_notifier_init(void) -{ - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); -} -core_initcall(scx_cgroup_lifetime_notifier_init); -#endif /* CONFIG_EXT_SUB_SCHED */ - static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) { static struct kthread_worker *helper; @@ -7903,6 +7942,21 @@ static bool bpf_scx_is_valid_access(int off, int size, return btf_ctx_access(off, size, type, prog, info); } +/* common to both forms: only scx.disallow is writable */ +static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg, + int off, int size) +{ + const struct btf_type *t; + + t = btf_type_by_id(reg->btf, reg->btf_id); + if (t == task_struct_type && + off >= offsetof(struct task_struct, scx.disallow) && + off + size <= offsetofend(struct task_struct, scx.disallow)) + return SCALAR_VALUE; + + return -EACCES; +} + static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size) @@ -7911,23 +7965,22 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(reg->btf, reg->btf_id); if (t == task_struct_type) { - /* - * COMPAT: Will be removed in v6.23. - */ if ((off >= offsetof(struct task_struct, scx.slice) && off + size <= offsetofend(struct task_struct, scx.slice)) || (off >= offsetof(struct task_struct, scx.dsq_vtime) && - off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { - pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n"); - return SCALAR_VALUE; - } - - if (off >= offsetof(struct task_struct, scx.disallow) && - off + size <= offsetofend(struct task_struct, scx.disallow)) + off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) return SCALAR_VALUE; } - return -EACCES; + return bpf_scx_btf_struct_access_common(reg, off, size); +} + +/* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */ +static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, int off, + int size) +{ + return bpf_scx_btf_struct_access_common(reg, off, size); } static const struct bpf_verifier_ops bpf_scx_verifier_ops = { @@ -7936,6 +7989,12 @@ static const struct bpf_verifier_ops bpf_scx_verifier_ops = { .btf_struct_access = bpf_scx_btf_struct_access, }; +static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = bpf_scx_is_valid_access, + .btf_struct_access = bpf_scx_cid_btf_struct_access, +}; + static int bpf_scx_init_member(const struct btf_type *t, const struct btf_member *member, void *kdata, const void *udata) @@ -7977,6 +8036,27 @@ static int bpf_scx_init_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, hotplug_seq): ops->hotplug_seq = *(u64 *)(udata + moff); return 1; + case offsetof(struct sched_ext_ops, cid_shard_size): + ops->cid_shard_size = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): { + u32 bw_ppt = *(u32 *)(udata + moff); + + if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE) + return -E2BIG; + ops->rescue_bandwidth_ppt = bw_ppt; + return 1; + } + case offsetof(struct sched_ext_ops, rescue_quantum_us): { + u32 quantum_us = *(u32 *)(udata + moff); + + if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US) + return -E2BIG; + if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US) + return -EINVAL; + ops->rescue_quantum_us = quantum_us; + return 1; + } #ifdef CONFIG_EXT_SUB_SCHED case offsetof(struct sched_ext_ops, sub_cgroup_id): ops->sub_cgroup_id = *(u64 *)(udata + moff); @@ -7987,20 +8067,6 @@ static int bpf_scx_init_member(const struct btf_type *t, return 0; } -#ifdef CONFIG_EXT_SUB_SCHED -static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(prog->aux); - if (unlikely(!sch)) - return; - - scx_error(sch, "dispatch recursion detected"); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - static int bpf_scx_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog) @@ -8016,6 +8082,7 @@ static int bpf_scx_check_member(const struct btf_type *t, #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): + case offsetof(struct sched_ext_ops, init_cids): case offsetof(struct sched_ext_ops, init): case offsetof(struct sched_ext_ops, exit): case offsetof(struct sched_ext_ops, sub_attach): @@ -8039,6 +8106,11 @@ static int bpf_scx_check_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, dispatch): prog->aux->priv_stack_requested = true; prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; + break; + case offsetof(struct sched_ext_ops, sub_caps_updated): + prog->aux->priv_stack_requested = true; + prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated; + break; } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -8174,6 +8246,7 @@ static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} static void sched_ext_ops__cpu_online(s32 cpu) {} static void sched_ext_ops__cpu_offline(s32 cpu) {} +static s32 sched_ext_ops__init_cids(void) { return -EINVAL; } static s32 sched_ext_ops__init(void) { return -EINVAL; } static void sched_ext_ops__exit(struct scx_exit_info *info) {} static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} @@ -8215,6 +8288,7 @@ static struct sched_ext_ops __bpf_ops_sched_ext_ops = { .sub_detach = sched_ext_ops__sub_detach, .cpu_online = sched_ext_ops__cpu_online, .cpu_offline = sched_ext_ops__cpu_offline, + .init_cids = sched_ext_ops__init_cids, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, @@ -8238,11 +8312,13 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types - * identical, only param names differ across structs) are reused; only - * set_cmask needs a fresh stub since the second argument type differs. + * identical, only param names differ across structs) are reused. Some need + * fresh stubs, set_cmask due to an argument type difference and the sub-sched + * notifiers because no cpu-form stub exists to reuse. */ -static void sched_ext_ops_cid__set_cmask(struct task_struct *p, - const struct scx_cmask *cmask) {} +static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} +static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8264,19 +8340,22 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .enable = sched_ext_ops__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, + .cpuctl_init = sched_ext_ops__cgroup_init, + .cpuctl_exit = sched_ext_ops__cgroup_exit, + .cpuctl_prep_move = sched_ext_ops__cgroup_prep_move, + .cpuctl_move = sched_ext_ops__cgroup_move, + .cpuctl_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cpuctl_set_weight = sched_ext_ops__cgroup_set_weight, + .cpuctl_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cpuctl_set_idle = sched_ext_ops__cgroup_set_idle, #endif .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, + .sub_caps_updated = sched_ext_ops__sub_caps_updated, + .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, + .init_cids = sched_ext_ops__init_cids, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, @@ -8290,7 +8369,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { * verified to match by the BUILD_BUG_ON checks in scx_init(). */ static struct bpf_struct_ops bpf_sched_ext_ops_cid = { - .verifier_ops = &bpf_scx_verifier_ops, + .verifier_ops = &bpf_scx_cid_verifier_ops, .reg = bpf_scx_reg_cid, .unreg = bpf_scx_unreg, .check_member = bpf_scx_check_member, @@ -8354,21 +8433,23 @@ static bool can_skip_idle_kick(struct rq *rq) * We can skip idle kicking if @rq is going to go through at least one * full SCX scheduling cycle before going idle. Just checking whether * curr is not idle is insufficient because we could be racing - * balance_one() trying to pull the next task from a remote rq, which + * dispatch_one() trying to pull the next task from a remote rq, which * may fail, and @rq may become idle afterwards. * * The race window is small and we don't and can't guarantee that @rq is * only kicked while idle anyway. Skip only when sure. */ - return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); + return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH); } -static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) +static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq, + unsigned long *ksyncs) { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; const struct sched_class *cur_class; bool should_wait = false; + bool kickable; unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); @@ -8378,47 +8459,68 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) * During CPU hotplug, a CPU may depend on kicking itself to make * forward progress. Allow kicking self regardless of online state. If * @cpu is running a higher class task, we have no control over @cpu. - * Skip kicking. + * Skip kicking. A sub-sched lacking baseline access on @cid has no + * business forcing a reschedule there - skip. This is the authoritative + * cap check: ecaps is read here under @rq's lock. */ - if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !sched_class_above(cur_class, &ext_sched_class)) { - if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { - if (cur_class == &ext_sched_class) - rq->curr->scx.slice = 0; - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + kickable = (cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !sched_class_above(cur_class, &ext_sched_class); + + if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { + if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { + if (cur_class == &ext_sched_class) { + u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); + + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); + else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); + } + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } - if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { + if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; should_wait = true; } - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } resched_curr(rq); } else { - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + /* a kickable cpu was skipped solely for the missing caps */ + if (kickable) + __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); return should_wait; } -static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) +static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu, + struct rq *this_rq) { struct rq *rq = cpu_rq(cpu); unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); + /* idle kicks need baseline access too, see kick_one_cpu() */ if (!can_skip_idle_kick(rq) && - (cpu_online(cpu) || cpu == cpu_of(this_rq))) - resched_curr(rq); + (cpu_online(cpu) || cpu == cpu_of(this_rq))) { + if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE))) + resched_curr(rq); + else + __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); + } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); } @@ -8427,6 +8529,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) struct rq *this_rq = this_rq(); struct scx_rq *this_scx = &this_rq->scx; struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); + struct scx_sched_pcpu *pcpu, *tmp; bool should_wait = false; unsigned long *ksyncs; s32 cpu; @@ -8437,15 +8540,24 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; - for_each_cpu(cpu, this_scx->cpus_to_kick) { - should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } + /* + * Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to + * the list under local_irq_save() and only this irq_work consumes it. + * A plain list without locking is sufficient. + */ + list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { + list_del_init(&pcpu->to_kick_node); - for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { - kick_one_cpu_if_idle(cpu, this_rq); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + for_each_cpu(cpu, pcpu->cpus_to_kick) { + should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); + } + + for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) { + kick_one_cpu_if_idle(cpu, pcpu, this_rq); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); + } } /* @@ -8456,6 +8568,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(this_rq); this_scx->kick_sync_pending = true; resched_curr(this_rq); + scx_rq_lock_drop(this_rq); raw_spin_rq_unlock(this_rq); } } @@ -8565,16 +8678,17 @@ void __init init_sched_ext_class(void) int n = cpu_to_node(cpu); /* local_dsq's sch will be set during scx_root_enable() */ - BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); + BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED + BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); + scx_rescue_init(rq); +#endif INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); + INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick); raw_spin_lock_init(&rq->scx.deferred_reenq_lock); INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); @@ -8603,6 +8717,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) bool is_local = dsq_id == SCX_DSQ_LOCAL || (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; + if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { + scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); + return false; + } + if (*enq_flags & SCX_ENQ_IMMED) { if (unlikely(!is_local)) { scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); @@ -8612,6 +8731,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) *enq_flags |= SCX_ENQ_IMMED; } + if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) { + scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id); + return false; + } + return true; } @@ -8625,11 +8749,6 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p return false; } - if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { - scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); - return false; - } - /* see SCX_EV_INSERT_NOT_OWNED definition */ if (unlikely(!scx_task_on_sched(sch, p))) { __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); @@ -8643,14 +8762,14 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p } static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 enq_flags) + u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; struct task_struct *ddsp_task; ddsp_task = __this_cpu_read(direct_dispatch_task); if (ddsp_task) { - mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); + mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags); return; } @@ -8663,6 +8782,8 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, .task = p, .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, .dsq_id = dsq_id, + .slice = slice, + .vtime = vtime, .enq_flags = enq_flags, }; } @@ -8670,7 +8791,7 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, __bpf_kfunc_start_defs(); /** - * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ + * scx_bpf_dsq_insert___v2 - Insert a task into the FIFO queue of a DSQ * @p: task_struct to insert * @dsq_id: DSQ to insert into * @slice: duration @p can run for in nsecs, 0 to keep the current value @@ -8684,7 +8805,7 @@ __bpf_kfunc_start_defs(); * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch * and @p must match the task being enqueued. * - * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p + * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p * will be directly inserted into the corresponding dispatch queue after * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be * inserted into the local DSQ of the CPU returned by ops.select_cpu(). @@ -8723,12 +8844,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); + scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags); return true; } @@ -8749,14 +8865,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - p->scx.dsq_vtime = vtime; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ); return true; } @@ -8861,13 +8970,13 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { }; static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, - struct task_struct *p, u64 dsq_id, u64 enq_flags) + struct task_struct *p, u64 dsq_id, u64 enq_flags, + bool priq) { struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct scx_sched *sch; - struct rq *this_rq, *src_rq, *locked_rq; + struct rq *p_rq, *src_rq, *locked_rq; bool dispatched = false; - bool in_balance; unsigned long flags; /* @@ -8883,9 +8992,13 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) return false; + /* internal bit, can only go in after @enq_flags is vetted */ + if (priq) + enq_flags |= SCX_ENQ_DSQ_PRIQ; + /* * If the BPF scheduler keeps calling this function repeatedly, it can - * cause similar live-lock conditions as consume_dispatch_q(). + * cause similar live-lock conditions as scx_consume_dispatch_q(). */ if (unlikely(READ_ONCE(sch->aborting))) return false; @@ -8897,24 +9010,28 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, } /* - * Can be called from either ops.dispatch() locking this_rq() or any - * context where no rq lock is held. If latter, lock @p's task_rq which - * we'll likely need anyway. + * Can be called from either ops.dispatch() holding the dispatched rq's + * lock or any context where no rq lock is held. If latter, lock @p's + * task_rq which we'll likely need anyway. */ src_rq = task_rq(p); local_irq_save(flags); - this_rq = this_rq(); - in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; - if (in_balance) { - if (this_rq != src_rq) - switch_rq_lock(this_rq, src_rq); + /* + * Under core scheduling, dispatch can run for a sibling rq, so the + * locked rq is not necessarily this CPU's. + */ + locked_rq = scx_locked_rq(); + + if (locked_rq) { + if (locked_rq != src_rq) + switch_rq_lock(locked_rq, src_rq); } else { raw_spin_rq_lock(src_rq); } - locked_rq = src_rq; + p_rq = src_rq; raw_spin_lock(&src_dsq->lock); /* did someone else get to it while we dropped the locks? */ @@ -8924,27 +9041,28 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, } /* @p is still on $src_dsq and stable, determine the destination */ - dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); + dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); /* - * Apply vtime and slice updates before moving so that the new time is - * visible before inserting into $dst_dsq. @p is still on $src_dsq but - * this is safe as we're locking it. + * Apply vtime and slice updates before moving. @p is still on $src_dsq + * with both $src_dsq and its task_rq locked, satisfying the write + * rules, and the PRIQ insertion into $dst_dsq reads the new vtime. */ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) - p->scx.slice = kit->slice; + scx_set_task_slice(p, kit->slice); /* execute move */ - locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); + p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (in_balance) { - if (this_rq != locked_rq) - switch_rq_lock(locked_rq, this_rq); + if (locked_rq) { + if (locked_rq != p_rq) + switch_rq_lock(p_rq, locked_rq); } else { - raw_spin_rq_unlock_irqrestore(locked_rq, flags); + scx_rq_lock_drop(p_rq); + raw_spin_rq_unlock_irqrestore(p_rq, flags); } kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | @@ -9000,7 +9118,7 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) } /** - * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ + * scx_bpf_dsq_move_to_local___v2 - move a task from a DSQ to the current CPU's local DSQ * @dsq_id: DSQ to move task from. Must be a user-created DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * @enq_flags: %SCX_ENQ_* @@ -9042,7 +9160,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - flush_dispatch_buf(sch, dspc->rq); + scx_flush_dispatch_buf(sch, dspc->rq); dsq = find_user_dsq(sch, dsq_id); if (unlikely(!dsq)) { @@ -9050,11 +9168,11 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, return false; } - if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { + if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched - * tasks. Bump nr_tasks to tell balance_one() to retry on empty + * tasks. Bump nr_tasks to tell dispatch_one() to retry on empty * local DSQ. */ dspc->nr_tasks++; @@ -9139,7 +9257,7 @@ __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags); + p, dsq_id, enq_flags, false); } /** @@ -9164,48 +9282,9 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + p, dsq_id, enq_flags, true); } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler - * @cgroup_id: cgroup ID of the child scheduler to dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Allows a parent scheduler to trigger dispatching on one of its direct - * child schedulers. The child scheduler runs its dispatch operation to - * move tasks from dispatch queues to the local runqueue. - * - * Returns: true on success, false if cgroup_id is invalid, not a direct - * child, or caller lacks dispatch permission. - */ -__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) -{ - struct rq *this_rq = this_rq(); - struct scx_sched *parent, *child; - - guard(rcu)(); - parent = scx_prog_sched(aux); - if (unlikely(!parent)) - return false; - - child = scx_find_sub_sched(cgroup_id); - - if (unlikely(!child)) - return false; - - if (unlikely(scx_parent(child) != parent)) { - scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", - cgroup_id); - return false; - } - - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_dispatch) @@ -9296,10 +9375,10 @@ __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_a return -ENOMEM; /* - * init_dsq() must be called in GFP_KERNEL context. Init it with NULL - * @sch and update afterwards. + * scx_init_dsq() must be called in GFP_KERNEL context. Init it with + * NULL @sch and update afterwards. */ - ret = init_dsq(dsq, dsq_id, NULL); + ret = scx_init_dsq(dsq, dsq_id, NULL); if (ret) { kfree(dsq); return ret; @@ -9353,20 +9432,60 @@ __bpf_kfunc_start_defs(); * @slice: time slice to set in nsecs * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Set @p's time slice to @slice. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. + * Set @p's time slice. @p must be on the calling scheduler. The value is + * applied whether or not the caller holds @p's rq lock - see the slice write + * rules above for the ownership model. + * + * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on + * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is + * always allowed. On the stashed path the slice is packed into an atomic64_t + * with the scheduler id and a flag bit, so a slice too large to fit is clamped + * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. + * + * Return %true on success, %false if @p is not on the calling scheduler. */ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, const struct bpf_prog_aux *aux) { struct scx_sched *sch; + struct rq *locked_rq; guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false; - p->scx.slice = slice; + /* + * Directly write only when we hold the lock of the rq @p is queued or + * running on. See the write rules above. + * + * While @p is queued on a user DSQ or in the BPF scheduler, + * synchronization is the scheduler's responsibility. This write can + * race a concurrent dispatch's commit, see apply_slice_vtime(). + * + * Making this kfunc always go through the oob stash would leave the + * commit as the only direct writer and close the race, but that would + * require two more oob application points - the dispatch keep-prev test + * and the tick-time expiry check. + */ + locked_rq = scx_locked_rq(); + if (!locked_rq || + (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && + !task_current(locked_rq, p))) { + set_task_slice_oob(sch, p, slice); + return true; + } + + /* under the rq lock: apply now, extensions gated on baseline access */ + if (slice > p->scx.slice && + unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + return true; + } + + if (unlikely(!scx_set_task_slice(p, slice))) + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + return true; } @@ -9393,14 +9512,25 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { + struct scx_sched_pcpu *pcpu; struct rq *this_rq; unsigned long irq_flags; + /* + * The per-cpu kick list is guarded only by local_irq_save(), which does + * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. + */ + if (unlikely(in_nmi())) { + scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); + return; + } + local_irq_save(irq_flags); this_rq = this_rq(); + pcpu = this_cpu_ptr(sch->pcpu); /* * While bypassing for PM ops, IRQ handling may not be online which can @@ -9414,6 +9544,9 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting * rq locks. We can probably be smarter and avoid bouncing if called * from ops which don't hold a rq lock. + * + * The kick masks are owned by @sch->pcpu, so that a preempt kick can be + * attributed to @sch. */ if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); @@ -9423,21 +9556,25 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (raw_spin_rq_trylock(target_rq)) { if (can_skip_idle_kick(target_rq)) { + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); goto out; } + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); } - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); + cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); if (flags & SCX_KICK_PREEMPT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); if (flags & SCX_KICK_WAIT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); + cpumask_set_cpu(cpu, pcpu->cpus_to_wait); } + if (list_empty(&pcpu->to_kick_node)) + list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick); irq_work_queue(&this_rq->scx.kick_cpus_irq_work); out: local_irq_restore(irq_flags); @@ -9470,10 +9607,13 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, - * -errno otherwise. + * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the + * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a + * kick is dropped if the caller lacks baseline access on @cid, and a + * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks + * %SCX_CAP_PREEMPT for a task outside its subtree. */ -__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { struct scx_sched *sch; s32 cpu; @@ -9481,12 +9621,11 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux * guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) - return -ENODEV; + return; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) - return cpu; + return; scx_kick_cpu(sch, cpu, flags); - return 0; } /** @@ -9496,6 +9635,10 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux * * * Return the number of tasks in the DSQ matching @dsq_id. If not found, * -%ENOENT is returned. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) { @@ -9512,7 +9655,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux } if (dsq_id == SCX_DSQ_LOCAL) { - ret = READ_ONCE(this_rq()->scx.local_dsq.nr); + ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); @@ -9691,10 +9834,15 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - User DSQs * * Re-enqueues are performed asynchronously. Can be called from anywhere. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux) { + struct rq *locked_rq = scx_locked_rq(); struct scx_sched *sch; struct scx_dispatch_q *dsq; @@ -9713,12 +9861,12 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); - schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); + dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); + schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); } /** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Iterate over all of the tasks currently enqueued on the local DSQ of the @@ -9773,12 +9921,38 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, return ret; } -__printf(3, 0) -static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz) +/* + * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit + * is claimed first and the reason is formatted directly into the winner-owned + * exit_info buffer, which allows use from any context including NMI. + * + * @fmt_blame is the sched blamed for formatting failures through the + * scx_error() calls in __bstr_format() and differs from @sch when a parent + * supplies the kill reason for a child. A formatting failure doesn't revert the + * claim - @sch still exits with the claimed kind and a fallback message. + */ +__printf(5, 0) +bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, struct scx_sched *fmt_blame, char *fmt, + unsigned long long *data, u32 data__sz) { - return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), - fmt, data, data__sz); + struct scx_exit_info *ei = sch->exit_info; + u64 data_buf[MAX_BPRINTF_VARARGS]; + s32 ret; + + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) + return false; + + ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN, + fmt, data, data__sz); + if (ret < 0) + scnprintf(ei->msg, SCX_EXIT_MSG_LEN, + "exit message formatting failed (%d)", ret); + + scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id()); + return true; } __bpf_kfunc_start_defs(); @@ -9800,14 +9974,13 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, const struct bpf_prog_aux *aux) { struct scx_sched *sch; - unsigned long flags; - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + guard(rcu)(); + sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); + if (likely(sch)) + scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt, + data, data__sz); } /** @@ -9825,14 +9998,13 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux) { struct scx_sched *sch; - unsigned long flags; - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + guard(rcu)(); + sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); + if (likely(sch)) + scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data, + data__sz); } /** @@ -9872,8 +10044,8 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, sizeof(buf->line) - dd->cursor, fmt, data, data__sz); if (ret < 0) { - dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", - dd->prefix, fmt, data, data__sz, ret); + scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", + dd->prefix, fmt, data, data__sz, ret); return; } @@ -9991,6 +10163,62 @@ __bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) return arch_scale_freq_capacity(cpu); } +/* validate and apply a cpuperf target, see scx_bpf_cpuperf_set() */ +static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) +{ + struct rq *rq, *locked_rq; + struct rq_flags rf; + s32 ret; + + if (unlikely(perf > SCX_CPUPERF_ONE)) { + scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); + return -EINVAL; + } + + if (!scx_cpu_valid(sch, cpu, NULL)) + return -EINVAL; + + rq = cpu_rq(cpu); + locked_rq = scx_locked_rq(); + + /* + * When called with an rq lock held, restrict the operation to the + * corresponding CPU to prevent ABBA deadlocks. + */ + if (locked_rq && rq != locked_rq) { + scx_error(sch, "Invalid target CPU %d", cpu); + return -EINVAL; + } + + /* + * If no rq lock is held, allow to operate on any CPU by acquiring + * the corresponding rq lock. + */ + if (!locked_rq) { + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + } + + /* + * ecaps updates are folded under the rq lock, making this test + * authoritative: a write can never land after a revoke has taken + * effect on @cpu. + */ + if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) { + rq->scx.cpuperf_target = perf; + cpufreq_update_util(rq, 0); + ret = 0; + } else { + __scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1); + ret = -EACCES; + } + + if (!locked_rq) + rq_unlock_irqrestore(rq, &rf); + + return ret; +} + /** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest @@ -10016,39 +10244,7 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; - if (unlikely(perf > SCX_CPUPERF_ONE)) { - scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); - return; - } - - if (scx_cpu_valid(sch, cpu, NULL)) { - struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); - struct rq_flags rf; - - /* - * When called with an rq lock held, restrict the operation - * to the corresponding CPU to prevent ABBA deadlocks. - */ - if (locked_rq && rq != locked_rq) { - scx_error(sch, "Invalid target CPU %d", cpu); - return; - } - - /* - * If no rq lock is held, allow to operate on any CPU by - * acquiring the corresponding rq lock. - */ - if (!locked_rq) { - rq_lock_irqsave(rq, &rf); - update_rq_clock(rq); - } - - rq->scx.cpuperf_target = perf; - cpufreq_update_util(rq, 0); - - if (!locked_rq) - rq_unlock_irqrestore(rq, &rf); - } + scx_cpuperf_set(sch, cpu, perf); } /** @@ -10057,10 +10253,13 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au * @perf: target performance level [0, %SCX_CPUPERF_ONE] * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * cid-addressed equivalent of scx_bpf_cpuperf_set(). + * cid-addressed equivalent of scx_bpf_cpuperf_set(). A sub-sched needs + * SCX_CAP_PERF on @cid. Returns 0 if the target was applied, -%EACCES if + * the write was denied for missing caps, other -errnos if @cid didn't + * resolve. */ -__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, - const struct bpf_prog_aux *aux) +__bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, + const struct bpf_prog_aux *aux) { struct scx_sched *sch; s32 cpu; @@ -10069,11 +10268,12 @@ __bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) - return; + return -ENODEV; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) - return; - scx_bpf_cpuperf_set(cpu, perf, aux); + return cpu; + + return scx_cpuperf_set(sch, cpu, perf); } /** @@ -10125,13 +10325,15 @@ __bpf_kfunc u32 scx_bpf_nr_online_cids(void) * * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. * The current cpu is trivially valid, so this is just a table lookup. Return - * -EINVAL if called from a non-SCX program before any scheduler has ever - * been enabled (the cid table is still unallocated at that point). + * -EINVAL if called before any scheduler has ever published its cid tables. */ __bpf_kfunc s32 scx_bpf_this_cid(void) { - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + s16 *tbl; + guard(rcu)(); + + tbl = rcu_dereference(scx_cpu_to_cid_tbl); if (!tbl) return -EINVAL; return tbl[raw_smp_processor_id()]; @@ -10190,46 +10392,22 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) * @p: task of interest * * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a - * valid cpu, so this is just a table lookup. Return -EINVAL if called from - * a non-SCX program before any scheduler has ever been enabled. + * valid cpu, so this is just a table lookup. Return -EINVAL if called before + * any scheduler has ever published its cid tables. */ __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) { - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + s16 *tbl; + /* KF_RCU covers only @p - a sleepable program holds no RCU lock */ + guard(rcu)(); + + tbl = rcu_dereference(scx_cpu_to_cid_tbl); if (!tbl) return -EINVAL; return tbl[task_cpu(p)]; } -/** - * scx_bpf_cpu_rq - Fetch the rq of a CPU - * @cpu: CPU of the rq - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - if (!sch->warned_deprecated_rq) { - printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " - "use scx_bpf_locked_rq() when holding rq lock " - "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); - sch->warned_deprecated_rq = true; - } - - return cpu_rq(cpu); -} - /** * scx_bpf_locked_rq - Return the rq currently locked by SCX * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs @@ -10332,6 +10510,27 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) return container_of(scx, struct task_struct, scx); } +u64 __scx_bpf_now(struct rq *rq) +{ + /* the caller must be on @rq's cpu or hold its lock */ + lockdep_assert((rq == this_rq() && !preemptible()) || + lockdep_is_held(__rq_lockp(rq))); + + if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { + /* if the rq clock is valid, use the cached rq clock */ + return READ_ONCE(rq->scx.clock); + } else { + /* + * Otherwise, return a fresh rq clock. + * + * The rq clock is updated outside of the rq lock. + * In this case, keep the updated rq clock invalid so the next + * read outside the rq lock gets a fresh rq clock. + */ + return sched_clock_cpu(cpu_of(rq)); + } +} + /** * scx_bpf_now - Returns a high-performance monotonically non-decreasing * clock for the current CPU. The clock returned is in nanoseconds. @@ -10362,76 +10561,47 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) */ __bpf_kfunc u64 scx_bpf_now(void) { - struct rq *rq; - u64 clock; - - preempt_disable(); - - rq = this_rq(); - if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { - /* - * If the rq clock is valid, use the cached rq clock. - * - * Note that scx_bpf_now() is re-entrant between a process - * context and an interrupt context (e.g., timer interrupt). - * However, we don't need to consider the race between them - * because such race is not observable from a caller. - */ - clock = READ_ONCE(rq->scx.clock); - } else { - /* - * Otherwise, return a fresh rq clock. - * - * The rq clock is updated outside of the rq lock. - * In this case, keep the updated rq clock invalid so the next - * kfunc call outside the rq lock gets a fresh rq clock. - */ - clock = sched_clock_cpu(cpu_of(rq)); - } - - preempt_enable(); - - return clock; + /* + * Note that scx_bpf_now() is re-entrant between a process context and + * an interrupt context (e.g., timer interrupt). However, we don't need + * to consider the race between them because such race is not observable + * from a caller. + */ + guard(preempt)(); + return __scx_bpf_now(this_rq()); } static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) { - struct scx_event_stats *e_cpu; int cpu; /* Aggregate per-CPU event counters into @events. */ memset(events, 0, sizeof(*events)); for_each_possible_cpu(cpu) { - e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; - scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); - scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); - scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); - scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); + struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; +#define SCX_EVENT(name) (events->name += READ_ONCE(e_cpu->name)) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT } } -/* - * scx_bpf_events - Get a system-wide event counter to +/** + * scx_bpf_events - Read the event counters of the calling scheduler * @events: output buffer from a BPF program - * @events__sz: @events len, must end in '__sz'' for the verifier + * @events__sz: @events len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Read the event counters of the scheduler associated with the calling program. + * @events is zeroed when no scheduler can be resolved. */ -__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, - size_t events__sz) +__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz, + const struct bpf_prog_aux *aux) { struct scx_sched *sch; struct scx_event_stats e_sys; rcu_read_lock(); - sch = rcu_dereference(scx_root); + sch = scx_prog_sched(aux); if (sch) scx_read_events(sch, &e_sys); else @@ -10520,16 +10690,19 @@ BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_now) -BTF_ID_FLAGS(func, scx_bpf_events) +BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) #endif +BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_any) static const struct btf_kfunc_id_set scx_kfunc_set_any = { @@ -10555,7 +10728,6 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) @@ -10588,7 +10760,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) */ enum scx_kf_allow_flags { SCX_KF_ALLOW_UNLOCKED = 1 << 0, - SCX_KF_ALLOW_INIT = 1 << 1, + SCX_KF_ALLOW_INIT_CIDS = 1 << 1, SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, SCX_KF_ALLOW_DISPATCH = 1 << 3, SCX_KF_ALLOW_ENQUEUE = 1 << 4, @@ -10618,9 +10790,11 @@ static const u32 scx_kf_allow_flags[] = { #endif /* CONFIG_EXT_GROUP_SCHED */ [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, + [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, + [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, }; @@ -10635,7 +10809,7 @@ static const u32 scx_kf_allow_flags[] = { int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) { bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); - bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); + bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id); bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); @@ -10643,19 +10817,20 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); + bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id); u32 moff, flags; /* Not an SCX kfunc - allow. */ - if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || - in_cpu_release || in_idle || in_any)) + if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch || + in_cpu_release || in_idle || in_any || in_cid)) return 0; /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ if (prog->type == BPF_PROG_TYPE_SYSCALL) - return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; + return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle) ? 0 : -EACCES; + return (in_any || in_idle || in_cid) ? 0 : -EACCES; /* * add_subprog_and_kfunc() collects all kfunc calls, including dead code @@ -10690,7 +10865,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) return -EACCES; /* SCX struct_ops: check the per-op allow list. */ - if (in_any || in_idle) + if (in_any || in_idle || in_cid) return 0; moff = prog->aux->attach_st_ops_member_off; @@ -10698,7 +10873,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) return 0; - if ((flags & SCX_KF_ALLOW_INIT) && in_init) + if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids) return 0; if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) return 0; @@ -10732,6 +10907,9 @@ static int __init scx_init(void) CID_OFFSET_MATCH(timeout_ms, timeout_ms); CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); + CID_OFFSET_MATCH(cid_shard_size, cid_shard_size); + CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt); + CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us); CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); /* shared callbacks: the union view requires byte-for-byte offset match */ CID_OFFSET_MATCH(enqueue, enqueue); @@ -10754,28 +10932,31 @@ static int __init scx_init(void) CID_OFFSET_MATCH(dump_task, dump_task); CID_OFFSET_MATCH(sub_attach, sub_attach); CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); + CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); + CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); -#ifdef CONFIG_EXT_GROUP_SCHED - CID_OFFSET_MATCH(cgroup_init, cgroup_init); - CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); - CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); - CID_OFFSET_MATCH(cgroup_move, cgroup_move); - CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); - CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); - CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); - CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); -#endif /* renamed callbacks must occupy the same slot as their cpu-form sibling */ CID_OFFSET_MATCH(select_cpu, select_cid); CID_OFFSET_MATCH(set_cpumask, set_cmask); CID_OFFSET_MATCH(cpu_online, cid_online); CID_OFFSET_MATCH(cpu_offline, cid_offline); CID_OFFSET_MATCH(dump_cpu, dump_cid); +#ifdef CONFIG_EXT_GROUP_SCHED + CID_OFFSET_MATCH(cgroup_init, cpuctl_init); + CID_OFFSET_MATCH(cgroup_exit, cpuctl_exit); + CID_OFFSET_MATCH(cgroup_prep_move, cpuctl_prep_move); + CID_OFFSET_MATCH(cgroup_move, cpuctl_move); + CID_OFFSET_MATCH(cgroup_cancel_move, cpuctl_cancel_move); + CID_OFFSET_MATCH(cgroup_set_weight, cpuctl_set_weight); + CID_OFFSET_MATCH(cgroup_set_bandwidth, cpuctl_set_bandwidth); + CID_OFFSET_MATCH(cgroup_set_idle, cpuctl_set_idle); +#endif /* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH(priv, priv); /* - * cid-form must end exactly at @priv - validate_ops() skips + * cid-form must end exactly at @priv - scx_validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields * past the BPF allocation would be UB. */ diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 6f93cc32b650..d2973fb3af6d 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -12,6 +12,7 @@ #include "internal.h" #include "cid.h" #include "idle.h" +#include "sub.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); @@ -732,6 +733,55 @@ static void update_builtin_idle(int cpu, bool idle) } } +/* + * Notify schedulers of an idle transition on @cpu's cid, delivering to every + * sched that holds %SCX_CAP_BASE on the cid (the root holds every cap). A real + * transition (@do_notify) reaches all holders. A forced one (@root_renotify for + * the root, a sub-sched's idle_renotify marker for a sub) reaches only the owed + * scheds. + */ +static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_renotify) +{ + s32 cpu = cpu_of(rq); + s32 cid = scx_cpu_arg(cpu); + struct scx_sched *root = scx_root_protected_live(); + struct scx_sched *pos; + + lockdep_assert_rq_held(rq); + + /* with no sub-sched, only the root can be owed a notification */ + if (!scx_has_subs()) { + if ((do_notify || root_renotify) && + SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu)) + SCX_CALL_OP(root, update_idle, rq, cid, idle); + return; + } + + pos = scx_next_descendant_pre(NULL, root); + while (pos) { + bool forced = false; + + if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) { + pos = scx_skip_subtree_pre(pos, root); + continue; + } + + if (!pos->level) { + forced = root_renotify; + } +#ifdef CONFIG_EXT_SUB_SCHED + else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) { + per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false; + forced = true; + } +#endif + if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) && + !scx_bypassing(pos, cpu)) + SCX_CALL_OP(pos, update_idle, rq, cid, idle); + pos = scx_next_descendant_pre(pos, root); + } +} + /* * Update the idle state of a CPU to @idle. * @@ -750,44 +800,39 @@ static void update_builtin_idle(int cpu, bool idle) */ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) { - struct scx_sched *sch = scx_root; int cpu = cpu_of(rq); lockdep_assert_rq_held(rq); /* - * Update the idle masks: - * - for real idle transitions (do_notify == true) - * - for idle-to-idle transitions (indicated by the previous task - * being the idle thread, managed by pick_task_idle()) - * - * Skip updating idle masks if the previous task is not the idle - * thread, since set_next_task_idle() has already handled it when - * transitioning from a task to the idle thread (calling this - * function with do_notify == true). - * - * In this way we can avoid updating the idle masks twice, - * unnecessarily. + * pick_task_idle() calls here only on an idle-to-idle re-pick and the + * transitions call with @do_notify, so every reaching call updates the + * masks. */ if (static_branch_likely(&scx_builtin_idle_enabled)) - if (do_notify || is_idle_task(rq->curr)) - update_builtin_idle(cpu, idle); + update_builtin_idle(cpu, idle); /* - * Trigger ops.update_idle() only when transitioning from a task to - * the idle thread and vice versa. + * ops.update_idle() fires on real idle transitions, indicated by + * @do_notify and managed by put_prev_task_idle()/set_next_task_idle(). + * An idle pick also fires it to flush a forced notify owed to a sched + * that missed transitions while bypassed or on a cid it just gained. + * unbypass_renotify_idle() and scx_process_sync_ecaps() arm the per-rq + * gates, and scx_idle_notify() targets the owed scheds. * - * Idle transitions are indicated by do_notify being set to true, - * managed by put_prev_task_idle()/set_next_task_idle(). - * - * This must come after builtin idle update so that BPF schedulers can - * create interlocking between ops.update_idle() and ops.enqueue() - + * This must come after the builtin idle update so that BPF schedulers + * can create interlocking between ops.update_idle() and ops.enqueue() - * either enqueue() sees the idle bit or update_idle() sees the task * that enqueue() queued. */ - if (SCX_HAS_OP(sch, update_idle) && do_notify && - !scx_bypassing(sch, cpu_of(rq))) - SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); + if (do_notify || + (idle && (rq->scx.flags & + (SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY)))) { + bool root_renotify = rq->scx.flags & SCX_RQ_ROOT_IDLE_RENOTIFY; + + rq->scx.flags &= ~(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY); + scx_idle_notify(rq, idle, do_notify, root_renotify); + } } static void reset_idle_masks(struct sched_ext_ops *ops) @@ -795,20 +840,20 @@ static void reset_idle_masks(struct sched_ext_ops *ops) int node; /* - * Consider all online cpus idle. Should converge to the actual state - * quickly. + * Start with all CPUs marked busy. The idle masks are populated when + * bypass is lifted and each idle CPU is forced through an idle re-pick. + * This may temporarily omit idle CPUs but never advertises a busy CPU as + * idle. */ if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); + cpumask_clear(idle_cpumask(NUMA_NO_NODE)->cpu); + cpumask_clear(idle_cpumask(NUMA_NO_NODE)->smt); return; } for_each_node(node) { - const struct cpumask *node_mask = cpumask_of_node(node); - - cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); - cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); + cpumask_clear(idle_cpumask(node)->cpu); + cpumask_clear(idle_cpumask(node)->smt); } } @@ -1318,7 +1363,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, /** * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu @@ -1365,7 +1410,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, * or pick any CPU from @node * @cpus_allowed: Allowed cpumask * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any @@ -1416,7 +1461,7 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, /** * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h new file mode 100644 index 000000000000..ed423bcc26b8 --- /dev/null +++ b/kernel/sched/ext/inlines.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Inline definitions layered on top of internal.h and cid.h. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_INLINES_H +#define _KERNEL_SCHED_EXT_INLINES_H + +#include "internal.h" +#include "cid.h" + +/* what dispatch concluded, consumed by the pick that follows */ +enum scx_dsp_verdict { + SCX_DSP_NONE, /* nothing to run */ + SCX_DSP_LOCAL, /* local DSQ has tasks */ + SCX_DSP_PREV, /* keep running @prev */ + SCX_DSP_RETRY, /* pick helpers only: restart the pick */ +}; + +/* + * One user of this function is scx_bpf_sub_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline enum scx_dsp_verdict +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (scx_consume_global_dsq(sch, rq)) + return SCX_DSP_LOCAL; + + if (scx_bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) { + if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; + return SCX_DSP_NONE; + } + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return SCX_DSP_LOCAL; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return SCX_DSP_NONE; + + dspc->rq = rq; + + /* + * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq + * lock, the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + +#ifdef CONFIG_EXT_SUB_SCHED + /* stash @prev so that nested invocations can access it */ + if (!nested) + rq->scx.sub_dispatch_prev = prev; +#endif + + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + +#ifdef CONFIG_EXT_SUB_SCHED + if (!nested) + rq->scx.sub_dispatch_prev = NULL; +#endif + + scx_flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) + return SCX_DSP_PREV; + if (rq->scx.local_dsq.nr) + return SCX_DSP_LOCAL; + if (scx_consume_global_dsq(sch, rq)) + return SCX_DSP_LOCAL; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * dispatch, we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (scx_bypass_dsp_enabled(sch) && + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; + + return SCX_DSP_NONE; +} + +#endif /* _KERNEL_SCHED_EXT_INLINES_H */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 673059fa9d72..27bbf5e04d90 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -11,6 +11,34 @@ #include "../sched.h" #include "types.h" +#include + +/** + * scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This can be used when preemption is not disabled. + */ +#define scx_add_event(sch, name, cnt) do { \ + this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, (cnt)); \ +} while(0) + +/** + * __scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This should be used only when preemption is disabled. + */ +#define __scx_add_event(sch, name, cnt) do { \ + __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, cnt); \ +} while(0) + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) @@ -23,10 +51,13 @@ enum scx_exit_kind { SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ SCX_EXIT_PARENT, /* parent exiting */ + SCX_EXIT_PARENT_KILL, /* killed by parent scheduler */ SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ + SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */ + SCX_EXIT_ERROR_RESCUE, /* ejected for overloading rescue execution */ }; /* @@ -372,8 +403,9 @@ struct sched_ext_ops { * @p: task running currently * * This operation is called every 1/HZ seconds on CPUs which are - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an - * immediate dispatch cycle on the CPU. + * executing an SCX task. Setting a slice of 0 for @p with + * scx_bpf_task_set_slice() will trigger an immediate dispatch cycle on + * the CPU. */ void (*tick)(struct task_struct *p); @@ -453,7 +485,7 @@ struct sched_ext_ops { * - sleeping (%SCX_DEQ_SLEEP) * - being moved to another CPU * - being temporarily taken off the queue for an attribute change - * (%SCX_DEQ_SAVE) + * (%SCX_DEQ_SCHED_CHANGE) * * This and ->dequeue() are related but not coupled. This operation * notifies @p's state transition and may not be preceded by ->dequeue() @@ -489,6 +521,11 @@ struct sched_ext_ops { * the BPF scheduler. Should return %true if @a should run before @b. * %false if there's no required ordering or @b should run before @a. * + * In a scheduler hierarchy, a pair spanning two schedulers is ordered + * by the nearest common ancestor implementing this op, so the op may be + * called on tasks that the scheduler delegated to its sub-schedulers + * and is not scheduling anymore. See scx_prio_less(). + * * If not specified, the default is ordering them according to when they * became runnable. */ @@ -615,8 +652,19 @@ struct sched_ext_ops { * @cgrp: cgroup being initialized * @args: init arguments, see the struct definition * - * Either the BPF scheduler is being loaded or @cgrp created, initialize - * @cgrp for sched_ext. This operation may block. + * Initialize @cgrp for sched_ext, delivered to @cgrp's sched either + * when the BPF scheduler is being loaded or when @cgrp is created. This + * operation may block. + * + * Cgroup handovers also generate these ops: an enabling sub-scheduler + * receives ops.cgroup_init() for every cgroup in its subtree while the + * previous sched receives ops.cgroup_exit(), and disabling reverses the + * two. + * + * When the BPF scheduler is being loaded or cgroups are being handed + * over, @cgrp may already have been removed by userspace: a removed + * cgroup stays schedulable until its dying tasks finish their final + * context switches. * * Return 0 for success, -errno for failure. An error return while * loading will abort loading of the BPF scheduler. During cgroup @@ -629,8 +677,13 @@ struct sched_ext_ops { * @cgroup_exit: Exit a cgroup * @cgrp: cgroup being exited * - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit - * @cgrp for sched_ext. This operation my block. + * Exit @cgrp for sched_ext, delivered to the sched whose + * ops.cgroup_init() it pairs with, either when the BPF scheduler is + * being unloaded or when @cgrp is destroyed. This operation may block. + * + * For a destroyed @cgrp, delivery follows the last scheduling event on + * it: a removed cgroup stays schedulable until its dying tasks finish + * their final context switches. */ void (*cgroup_exit)(struct cgroup *cgrp); @@ -643,6 +696,12 @@ struct sched_ext_ops { * Prepare @p for move from cgroup @from to @to. This operation may * block and can be used for allocations. * + * The cgroup_move ops are delivered to @p's sched, and only for moves + * that don't re-home @p. A re-homing move is reported through + * ops.exit_task() and ops.init_task() instead. @from and @to can + * reference cgroups the sched never received ops.cgroup_init() for, as + * the cpu controller can be coarser than the sub-scheduler topology. + * * Return 0 for success, -errno for failure. An error return aborts the * migration. */ @@ -678,6 +737,11 @@ struct sched_ext_ops { * @weight: new weight [1..10000] * * Update @cgrp's weight to @weight. + * + * Knobs of a cgroup belong to the parent, so the set_* ops are + * delivered to @cgrp's parent's sched. That sched may never have seen + * ops.cgroup_init() for @cgrp - at a sub-scheduler attach point, the + * parent sched tracks @cgrp through ops.sub_attach() instead. */ void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); @@ -698,6 +762,8 @@ struct sched_ext_ops { * burst temporarily. The specific control mechanism and thus the * interpretation of @period_us and burstiness is up to the BPF * scheduler. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_bandwidth)(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us); @@ -710,6 +776,8 @@ struct sched_ext_ops { * Update @cgrp's idle state to @idle. This callback is invoked when * a cgroup transitions between idle and non-idle states, allowing the * BPF scheduler to adjust its behavior accordingly. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); @@ -729,6 +797,39 @@ struct sched_ext_ops { */ void (*sub_detach)(struct scx_sub_detach_args *args); + /** + * @sub_caps_updated: Caps on this sub-sched's shard changed + * @cmask: cids whose caps changed (cmask->base identifies the shard) + * @caps: SCX_CAP_* that changed + * + * Invoked after grant or revoke modifies caps on a shard. There can be + * only one in-flight invocation per shard. @cmask and @caps coalesce + * all changes since the last delivery. Direction (set vs cleared) isn't + * encoded. Query current state with scx_bpf_sub_caps(). + * + * Delivered asynchronously after the change is recorded, and may run + * before it takes effect on any given cpu. Use it to track which caps + * the sub-sched holds and propagate to its own children, not to decide + * if a task can run on a cpu now. sub_ecaps_updated() reports that per + * cpu, once it is in effect. + * + * May call scx_bpf_sub_grant() / scx_bpf_sub_revoke() on children. + */ + void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); + + /** + * @sub_ecaps_updated: This sub-sched's effective caps on a cid changed + * @cid: the cid whose effective caps changed + * @before: effective caps as of the last delivery + * @after: effective caps now + * + * Invoked when this sub-sched's effective caps on @cid change, once the + * change is in effect on the cpu. Runs in dispatch context with rq lock + * held, and can perform all operations allowed in ops.dispatch() + * including inserting/moving tasks. + */ + void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -752,9 +853,18 @@ struct sched_ext_ops { void (*cpu_offline)(s32 cpu); /* - * All CPU hotplug ops must come before ops.init(). + * All CPU hotplug ops must come before ops.init_cids(). */ + /** + * @init_cids: Finalize the cid layout (cid-form only) + * + * Runs after the default cid layout is built, before caps and shards + * are finalized. A cid-form scheduler may call scx_bpf_cid_override() + * here for a custom layout. Ignored for cpu-form schedulers. + */ + s32 (*init_cids)(void); + /** * @init: Initialize the BPF scheduler */ @@ -809,8 +919,51 @@ struct sched_ext_ops { u64 hotplug_seq; /** - * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the - * specified cgroup. + * @cid_shard_size: Target number of CIDs per shard + * + * Shards are contiguous CID ranges used as operation and locking + * domains for sub-scheduling. Each LLC is divided into ceil(nr_cpus / + * @cid_shard_size) shards, then cores are distributed across them + * evenly. If one core has more logical CPUs than @cid_shard_size, its + * shard will become larger than @cid_shard_size. Values above + * SCX_CID_SHARD_MAX_CPUS are capped. 0 means use the default (24). + */ + u32 cid_shard_size; + + /** + * @rescue_bandwidth_ppt: Rescue execution bandwidth in parts per thousand + * + * The fraction of each CPU's time that may be consumed running tasks + * from its rescue DSQ. A higher bandwidth admits and escalates rescues + * faster, see @rescue_quantum_us. + * + * Only the root scheduler's value is used. 0 means the default of 20 + * (2%). May not exceed 250 (25%). %SCX_RESCUE_DISABLE disables rescue - + * %SCX_ENQ_RESCUE inserts are then rejected like any other insert + * lacking the caps. + */ + u32 rescue_bandwidth_ppt; + + /** + * @rescue_quantum_us: Rescue execution quantum in microseconds + * + * How much CPU time each rescue gets. Rescues run one at a time per CPU + * and admissions are paced to keep rescue execution within + * @rescue_bandwidth_ppt - with the defaults, one 5ms rescue every + * 250ms. A crowded queue round-robins on the quantum divided across the + * waiters, floored at 1ms. A stuck rescue eventually escalates to + * forced execution. A larger quantum interrupts the CPU less often but + * for longer and spaces rescues further apart. + * + * Only the root scheduler's value is used. 0 means the default (5000). + * Non-zero values must be within [1000, 100000]. Values too short for + * the kernel to meter are lifted silently. + */ + u32 rescue_quantum_us; + + /** + * @sub_cgroup_id: When >1, attach the scheduler as a sub-scheduler + * on the specified cgroup. */ u64 sub_cgroup_id; @@ -818,8 +971,9 @@ struct sched_ext_ops { * @name: BPF scheduler's name * * Must be a non-zero valid BPF object name including only isalnum(), - * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the - * BPF scheduler is enabled. + * '_' and '.' chars. Exposed via the ops file in the scheduler's sysfs + * directory, /sys/kernel/sched_ext/root/ops for the root scheduler, + * while the BPF scheduler is enabled. */ char name[SCX_OPS_NAME_LEN]; @@ -877,6 +1031,7 @@ struct sched_ext_ops { * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid + * - cgroup_* -> cpuctl_* (they track the cgroup cpu controller) * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) * * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; @@ -900,7 +1055,7 @@ struct sched_ext_ops_cid { struct task_struct *b); void (*set_weight)(struct task_struct *p, u32 weight); void (*set_cmask)(struct task_struct *p, - const struct scx_cmask *cmask); + const struct scx_cmask *cmask__arena); void (*update_idle)(s32 cid, bool idle); s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); @@ -912,24 +1067,25 @@ struct sched_ext_ops_cid { void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); #ifdef CONFIG_EXT_GROUP_SCHED - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - void (*cgroup_exit)(struct cgroup *cgrp); - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); + s32 (*cpuctl_init)(struct cgroup *cgrp, struct scx_cgroup_init_args *args); + void (*cpuctl_exit)(struct cgroup *cgrp); + s32 (*cpuctl_prep_move)(struct task_struct *p, struct cgroup *from, + struct cgroup *to); + void (*cpuctl_move)(struct task_struct *p, struct cgroup *from, struct cgroup *to); + void (*cpuctl_cancel_move)(struct task_struct *p, struct cgroup *from, + struct cgroup *to); + void (*cpuctl_set_weight)(struct cgroup *cgrp, u32 weight); + void (*cpuctl_set_bandwidth)(struct cgroup *cgrp, u64 period_us, u64 quota_us, + u64 burst_us); + void (*cpuctl_set_idle)(struct cgroup *cgrp, bool idle); #endif /* CONFIG_EXT_GROUP_SCHED */ s32 (*sub_attach)(struct scx_sub_attach_args *args); void (*sub_detach)(struct scx_sub_detach_args *args); + void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); + void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); + s32 (*init_cids)(void); s32 (*init)(void); void (*exit)(struct scx_exit_info *info); @@ -939,6 +1095,9 @@ struct sched_ext_ops_cid { u32 timeout_ms; u32 exit_dump_len; u64 hotplug_seq; + u32 cid_shard_size; + u32 rescue_bandwidth_ppt; + u32 rescue_quantum_us; u64 sub_cgroup_id; char name[SCX_OPS_NAME_LEN]; @@ -954,8 +1113,8 @@ enum scx_opi { SCX_OPI_NORMAL_BEGIN = 0, SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), - SCX_OPI_END = SCX_OP_IDX(init), + SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init_cids), + SCX_OPI_END = SCX_OP_IDX(init_cids), }; /* @@ -1001,15 +1160,13 @@ struct scx_event_stats { s64 SCX_EV_REENQ_IMMED; /* - * The number of times a reenq of local DSQ caused another reenq of - * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher - * priority class task even if the BPF scheduler always satisfies the - * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, - * that scenario is very unlikely and this count going up regularly - * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ - * incorrectly causing recursive reenqueues. + * The number of times a reenqueue (%SCX_ENQ_REENQ) led to another + * reenqueue without the task running in between. This count climbing + * rapidly indicates that the BPF scheduler keeps re-deciding placements + * it can't honor. A single task reenqueued more than + * %SCX_REENQ_MAX_REPEAT times gets its owning scheduler ejected. */ - s64 SCX_EV_REENQ_LOCAL_REPEAT; + s64 SCX_EV_REENQ_REPEAT; /* * Total number of times a task's time slice was refilled with the @@ -1017,6 +1174,18 @@ struct scx_event_stats { */ s64 SCX_EV_REFILL_SLICE_DFL; + /* + * The number of times an out-of-band slice request exceeded the maximum + * representable value and was clamped. + */ + s64 SCX_EV_SLICE_CLAMPED; + + /* + * The number of times a slice extension was denied because the + * scheduler lacked baseline cpu access on the task's cpu. + */ + s64 SCX_EV_SLICE_DENIED; + /* * The total duration of bypass modes in nanoseconds. */ @@ -1049,8 +1218,70 @@ struct scx_event_stats { * from sub_bypass_dsq's. */ s64 SCX_EV_SUB_BYPASS_DISPATCH; + + /* + * The number of times a migration-disabled task lacking the cap for its + * cid was allowed onto the local DSQ. It must run on its pinned CPU, so + * it can't be rejected. The violation is counted here. + */ + s64 SCX_EV_SUB_FORCED_ADMIT; + + /* + * The number of times a preempting kick was refused because the + * sub-sched lacked SCX_CAP_PREEMPT for a task outside its subtree. The + * kick degrades to a plain reschedule. + */ + s64 SCX_EV_SUB_PREEMPT_DENIED; + + /* + * The number of times a kick was skipped because the sub-sched lacked + * baseline access on the target cid. The preempt-part degradation of a + * delivered kick is counted in SCX_EV_SUB_PREEMPT_DENIED instead. + */ + s64 SCX_EV_SUB_KICK_DENIED; + + /* + * The number of times a local DSQ reenq was dropped because the + * sub-sched lacked baseline access on the target cid. + */ + s64 SCX_EV_SUB_REENQ_DENIED; + + /* + * The number of times scx_bpf_cidperf_set() was denied because the + * sub-sched lacked SCX_CAP_PERF on the target cid. + */ + s64 SCX_EV_SUB_CIDPERF_DENIED; + + /* + * The number of times an insert carrying %SCX_ENQ_RESCUE lacked the + * caps for its cid and the task entered the rescue path. + */ + s64 SCX_EV_SUB_RESCUE; }; +#define SCX_EVENTS_LIST(SCX_EVENT) \ + SCX_EVENT(SCX_EV_SELECT_CPU_FALLBACK); \ + SCX_EVENT(SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); \ + SCX_EVENT(SCX_EV_DISPATCH_KEEP_LAST); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); \ + SCX_EVENT(SCX_EV_REENQ_IMMED); \ + SCX_EVENT(SCX_EV_REENQ_REPEAT); \ + SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ + SCX_EVENT(SCX_EV_SLICE_CLAMPED); \ + SCX_EVENT(SCX_EV_SLICE_DENIED); \ + SCX_EVENT(SCX_EV_BYPASS_DURATION); \ + SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ + SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ + SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT); \ + SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ + SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \ + SCX_EVENT(SCX_EV_SUB_REENQ_DENIED); \ + SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED); \ + SCX_EVENT(SCX_EV_SUB_RESCUE) + struct scx_sched; enum scx_sched_pcpu_flags { @@ -1062,6 +1293,8 @@ struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; + u64 slice; + u64 vtime; u64 enq_flags; }; @@ -1075,14 +1308,55 @@ struct scx_dsp_ctx { struct scx_deferred_reenq_local { struct list_head node; u64 flags; - u64 seq; - u32 cnt; }; struct scx_sched_pcpu { struct scx_sched *sch; u64 flags; /* protected by rq lock */ + /* + * Kick state owned by this cpu for this sched. scx_kick_cpu() records + * targets here and links @to_kick_node onto the cpu's + * rq->scx.sched_pcpus_to_kick. The cpu's single kick irq_work walks + * that list and kicks each sched's targets on its behalf. Per-sched so + * a kick stays attributed to its scheduler. + */ + cpumask_var_t cpus_to_kick; + cpumask_var_t cpus_to_kick_if_idle; + cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_wait; + struct list_head to_kick_node; + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * pshard->caps[cap_bit] is the set of cids the sched holds that one + * cap on. ecaps is its transpose: the set of SCX_CAP_* bits the sched + * effectively holds on this cpu, with implied caps folded in, so that + * the hot-path check is a single read. + * + * While pshard->caps[] under pshard->lock is the target configuration, + * ecaps is the effective copy owned by the cpu. It is written under the + * rq lock while processing rq->ecaps_to_sync. Can also be read with + * READ_ONCE() outside rq lock. + * + * See queue_sync_ecaps() and scx_process_sync_ecaps(). + */ + u64 ecaps; + struct llist_node ecaps_to_sync_node; + /* owed a forced update_idle() re-notify on this cpu */ + bool idle_renotify; + /* effective caps as of the last sub_ecaps_updated() delivery */ + u64 reported_ecaps; + + /* + * Decaying rescue runtime consumed on this cpu, see + * scx_rescue_decay_avg(). Overload on this cpu ejects the sub with the + * largest value. Accessed only under this cpu's rq lock. + */ + u64 rescue_avg; + u64 rescue_avg_at; /* last decay, jiffies_64 */ +#endif + /* * The event counters are in a per-CPU variable to minimize the * accounting overhead. A system-wide view on the event counter is @@ -1104,6 +1378,146 @@ struct scx_sched_pnode { struct scx_dispatch_q global_dsq; }; +/* + * Sub-sched capability delegation. + * + * Caps are per-cid permissions parents delegate to direct children via + * scx_bpf_sub_grant() / scx_bpf_sub_revoke(). A child's cap set is always a + * subset of its parent's. A sub-sched checks its caps locally, and cross-sched + * communication is needed only when the delegation set itself changes. + * + * Caps are used to implement sub-sched scheduling on the enqueue path. Picking + * a cid for a task at a leaf depends on which cids the leaf is allowed to use. + * Resolving that programmatically on every enqueue would mean a cross-sched + * round-trip call chain, possibly retrying if the request can't be granted + * as-is. + * + * The dispatch path is different - it runs as top-down recursion via + * scx_bpf_sub_dispatch(): a sched's dispatch op invokes a child's dispatch op + * on the local rq, and the subtree dispatches in a single pass. + * + * Locking is per shard. cid space is split into shards, and each sub-sched has + * its own pshard->lock for each shard. Operations are broken up on shard + * boundaries. Different shards never contend. Shards are expected to be + * topology-aligned and likely to serve as the locality unit when cids are + * allocated to schedulers, so per-shard lock granularity scales naturally with + * the allocation pattern. + * + * ENQ_IMMED insert an IMMED task onto the cid's local DSQ + * - kick the cid's cpu (except SCX_KICK_PREEMPT) + * + * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) + * + * PREEMPT preempt any task running on the cid regardless of the owning + * sched (implies ENQ). Preempting a task in the sched's own subtree + * doesn't require any cap. + * - SCX_ENQ_PREEMPT inserts + * - SCX_KICK_PREEMPT kicks + * + * PERF control the cid's cpu power/perf management state, currently the + * cpufreq target set through scx_bpf_cidperf_set(). Hardware + * control is a separate axis from queue access: PERF neither + * implies nor is implied by the caps above. + * + * Implied caps apply to the holder's own use of a cid, not to delegation. + * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through + * implication is usable but cannot be re-delegated to a child. When granting a + * cap, it usually makes sense to delegate its implied caps explicitly alongside + * it. + */ +enum scx_cap_flags { + __SCX_CAP_ENQ_IMMED = 0, + __SCX_CAP_ENQ = 1, + __SCX_CAP_PREEMPT = 2, + __SCX_CAP_PERF = 3, + + __SCX_NR_CAPS, + __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, + + SCX_CAP_ENQ_IMMED = BIT_U64(__SCX_CAP_ENQ_IMMED), + SCX_CAP_ENQ = BIT_U64(__SCX_CAP_ENQ), + SCX_CAP_PREEMPT = BIT_U64(__SCX_CAP_PREEMPT), + SCX_CAP_PERF = BIT_U64(__SCX_CAP_PERF), + + /* alias for minimal cap to make any use of a cpu */ + SCX_CAP_BASE = SCX_CAP_ENQ_IMMED, + + /* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */ + SCX_CAPS_REENQ_ON_LOSS = SCX_CAP_ENQ_IMMED | SCX_CAP_ENQ, +}; + +#ifdef CONFIG_EXT_SUB_SCHED +/* iterate set bits in a u64 cap mask */ +#define scx_for_each_cap_bit(cap_bit, caps) \ + for (u64 __caps = (caps); \ + __caps && ((cap_bit) = __ffs64(__caps), true); \ + __caps &= __caps - 1) + +/* + * Sub-cap update notifier. + * + * ops_cid.sub_caps_updated() notifies sub-scheds when their cap state changes + * so they can refresh internal state without polling scx_bpf_sub_caps() per + * enqueue. + * + * Three constraints shape the design: + * + * 1. Static memory. Deliveries use a fixed-size buffer, both for runtime + * efficiency and so notifications can't be lost under memory pressure. + * + * 2. High-frequency updates. Grant/revoke can mutate caps in bursts, and the + * notifier path must absorb that without amplifying it. + * + * 3. Recursive grant/revoke from the callback. A child receiving a + * notification can call grant/revoke on its own children, which can + * cascade recursively down its subtree. + * + * (1) and (2) lead to coalescing into a fixed payload. Each delivery carries a + * single (cmask, caps) pair covering every change since the previous one. + * Direction (set vs cleared) isn't encoded as it doesn't fit in the fixed-size + * summary. The callback queries scx_bpf_sub_caps() for current state. Only one + * delivery is in flight per shard. Further changes fold into the same buffer + * and ship as the next callback, so a shard's callbacks fire in order. + * + * (3) leads to deferred delivery. Events accumulate during grant/revoke and are + * delivered after the shard lock is released. + */ +struct scx_caps_updated { + raw_spinlock_t lock; + u64 caps; + struct scx_cmask *cmask_arena_out; + struct list_head node_in_flight; + /* Kernel-side accumulator. Access as &cu->cmask. */ + TRAILING_OVERLAP(struct scx_cmask, cmask, bits, + u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; + ); +}; + +struct scx_pshard { + raw_spinlock_t lock; /* serializes caps */ + struct scx_sched *sch; /* backpointer */ + struct scx_caps_updated caps_updated; + + /* + * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps + * the trailing _bits[] storage. Access as &caps[i].cmask. See + * scx_sched_pcpu->ecaps. + */ + TRAILING_OVERLAP(struct scx_cmask, cmask, bits, + u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; + ) caps[__SCX_NR_CAPS]; + + /* + * Shard geometry captured at alloc. cmask_arena_out's own header is + * bpf-writable and the live shard range can change before the + * rcu-deferred free, so re-init and size cmask_arena_out from these + * trusted copies instead. + */ + u32 base; + u32 nr_cids; +}; +#endif + struct scx_sched { /* * cpu-form and cid-form ops share field offsets up to .priv (verified @@ -1117,6 +1531,7 @@ struct scx_sched { struct sched_ext_ops_cid ops_cid; }; bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ + bool dead; /* set after ops.exit(), gates scx_prog_sched() */ /* * Arena map auto-discovered from member progs at struct_ops attach. @@ -1125,8 +1540,7 @@ struct scx_sched { * * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added * at the kernel-side mapping address. @arena_kern_base is the start - * of the arena's kern_vm range. See scx_arena_to_kaddr() and - * scx_kaddr_to_arena(). + * of the arena's kern_vm range. See scx_arena_to_kaddr(). */ struct bpf_map *arena_map; struct gen_pool *arena_pool; @@ -1135,7 +1549,7 @@ struct scx_sched { /* * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and hands BPF its arena pointer via scx_kaddr_to_arena(). + * and passes it to the callback's __arena argument. */ struct scx_cmask * __percpu *set_cmask_scratch; @@ -1151,13 +1565,16 @@ struct scx_sched { */ struct rhashtable dsq_hash; struct scx_sched_pnode **pnode; +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_pshard **pshard; /* indexed by shard_idx */ +#endif struct scx_sched_pcpu __percpu *pcpu; u64 slice_dfl; u64 bypass_timestamp; s32 bypass_depth; - /* bypass dispatch path enable state, see bypass_dsp_enabled() */ + /* bypass dispatch path enable state, see scx_bypass_dsp_enabled() */ unsigned long bypass_dsp_claim; atomic_t bypass_dsp_enable_depth; @@ -1166,16 +1583,28 @@ struct scx_sched { u32 dsp_max_batch; s32 level; +#ifdef CONFIG_EXT_SUB_SCHED + /* + * pshard[] size captured at enable for the async RCU free path - + * scx_nr_cid_shards may be rewritten by a later enable's + * scx_cid_publish_tables() before free runs. While sch is active, use + * the global. + */ + u32 nr_pshards; +#endif + /* * Updates to the following warned bitfields can race causing RMW issues * but it doesn't really matter. */ bool warned_zero_slice:1; - bool warned_deprecated_rq:1; bool warned_unassoc_progs:1; struct list_head all; + /* unique instance id, monotonic and never reused */ + u64 id; + #ifdef CONFIG_EXT_SUB_SCHED struct rhash_head hash_node; @@ -1185,6 +1614,7 @@ struct scx_sched { char *cgrp_path; struct kset *sub_kset; + bool linked; /* on ->children, see scx_link_sched() */ bool sub_attached; #endif /* CONFIG_EXT_SUB_SCHED */ @@ -1203,9 +1633,11 @@ struct scx_sched { struct kthread_worker *helper; struct irq_work disable_irq_work; struct kthread_work disable_work; + struct irq_work propagate_exit_irq_work; /* see scx_claim_exit() */ struct timer_list bypass_lb_timer; cpumask_var_t bypass_lb_donee_cpumask; cpumask_var_t bypass_lb_resched_cpumask; + cpumask_var_t stall_cpus; struct rcu_work rcu_work; /* all ancestors including self */ @@ -1227,16 +1659,6 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } -/** - * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form - * @sch: scheduler whose arena hosts @kaddr - * @kaddr: kernel-side arena address, supplied by trusted kernel code - */ -static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) -{ - return (void *)((uintptr_t)kaddr - sch->arena_kern_base); -} - enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -1278,6 +1700,17 @@ enum scx_enq_flags { */ SCX_ENQ_IMMED = 1LLU << 33, + /* + * Only allowed on local DSQs. If the insert lacks the caps for the + * target cid, divert the task to the CPU's rescue path instead of + * rejecting and reenqueueing, e.g. when the task's affinity is + * restricted to cids the scheduler doesn't hold. The kernel runs + * rescued tasks on the target CPU. Rescue execution is guaranteed to + * make forward progress and is bandwidth-limited, see the + * rescue_bandwidth_ppt and rescue_quantum_us ops fields. + */ + SCX_ENQ_RESCUE = 1LLU << 34, + /* * The task being enqueued was previously enqueued on a DSQ, but was * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find @@ -1303,6 +1736,9 @@ enum scx_enq_flags { SCX_ENQ_DSQ_PRIQ = 1LLU << 57, SCX_ENQ_NESTED = 1LLU << 58, SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ + SCX_ENQ_IGNORE_CAPS = 1LLU << 60, /* admit to local DSQ ignoring caps */ + SCX_ENQ_APPLY_SLICE = 1LLU << 61, /* apply carried slice/vtime at insertion */ + SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; enum scx_deq_flags { @@ -1329,6 +1765,9 @@ enum scx_reenq_flags { /* low 16bits determine which tasks should be reenqueued */ SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ + /* internal: kernel-issued on cap revoke, not accepted from BPF */ + SCX_REENQ_CAP_REVOKE = 1LLU << 1, + __SCX_REENQ_FILTER_MASK = 0xffffLLU, __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, @@ -1374,6 +1813,7 @@ enum scx_kick_flags { enum scx_tg_flags { SCX_TG_ONLINE = 1U << 0, SCX_TG_INITED = 1U << 1, + SCX_TG_SUB_INIT = 1U << 2, /* see scx_cgroup_claim_subtree() */ }; enum scx_enable_state { @@ -1520,6 +1960,47 @@ enum scx_ops_state { #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) +/* + * SCX task iterator. + */ +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked_task; + struct rq *rq; + struct rq_flags rf; + u32 cnt; + bool list_locked; +#ifdef CONFIG_EXT_SUB_SCHED + struct cgroup *cgrp; + struct cgroup_subsys_state *css_pos; + struct css_task_iter css_iter; +#endif +}; + +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ + int ret; +}; + +/* string formatting from BPF */ +struct scx_bstr_buf { + u64 data[MAX_BPRINTF_VARARGS]; + char line[SCX_EXIT_MSG_LEN]; +}; + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); @@ -1540,12 +2021,159 @@ __printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, ...); +u32 scx_get_task_state(const struct task_struct *p); +void scx_set_task_state(struct task_struct *p, u32 state); +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); +void scx_task_iter_unlock(struct scx_task_iter *iter); +void scx_task_iter_stop(struct scx_task_iter *iter); +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +bool scx_set_task_slice(struct task_struct *p, u64 slice); +void scx_task_slice_ended(struct rq *rq, struct task_struct *p); +void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu); +void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags, struct scx_dispatch_q *src_dsq, + struct rq *dst_rq); +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags); +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); +bool scx_rq_online(struct rq *rq); +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); +s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); +__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +u64 __scx_bpf_now(struct rq *rq); +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq); +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, + struct cgroup *cgrp, bool fork); +void scx_enable_task(struct scx_sched *sch, struct task_struct *p); +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p); +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) +void scx_cgroup_lock(void); +void scx_cgroup_unlock(void); +#endif +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +void scx_disable_bypass_dsp(struct scx_sched *sch); +void scx_bypass(struct scx_sched *sch, bool bypass); +s32 scx_link_sched(struct scx_sched *sch); +void scx_unlink_sched(struct scx_sched *sch); +void scx_disable_dump(struct scx_sched *sch); +void scx_log_sched_disable(struct scx_sched *sch); +void scx_flush_disable_work(struct scx_sched *sch); +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent); +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); +int scx_sched_sysfs_add(struct scx_sched *sch); +bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor); +__printf(5, 0) bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, struct scx_sched *fmt_blame, + char *fmt, unsigned long long *data, u32 data__sz); + +extern raw_spinlock_t scx_sched_lock; +extern struct mutex scx_enable_mutex; +extern struct percpu_rw_semaphore scx_fork_rwsem; +extern bool scx_cgroup_enabled; +extern struct list_head scx_sched_all; +#ifdef CONFIG_EXT_SUB_SCHED +extern const struct rhashtable_params scx_sched_hash_params; +extern struct rhashtable scx_sched_hash; +extern struct scx_sched *scx_enabling_sub_sched; +#endif + #define scx_exit(sch, kind, exit_code, fmt, args...) \ __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) + +/** + * scx_root_protected_live - Root sched for paths that only run while live + * + * scx_root is published before the scheduler goes live and cleared only after + * it is fully drained, so a path that only executes while the scheduler is live + * can never race an update. Return the root sched with a plain load, never + * %NULL. + */ +static inline struct scx_sched *scx_root_protected_live(void) +{ + return rcu_dereference_protected(scx_root, true); +} + +/** + * scx_root_protected - Root sched for contexts that exclude its updates + * + * Both scx_root updates run under the locks checked below, so holding one + * excludes them. Return the root sched with a plain load, %NULL if no scheduler + * is loaded. + */ +static inline struct scx_sched *scx_root_protected(void) +{ + return rcu_dereference_protected(scx_root, + lockdep_is_cpus_held() || + lockdep_is_held(&scx_enable_mutex)); +} + +static inline struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) +{ + return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; +} + +/** + * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled + * @sch: scheduler to check + * + * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled + * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors + * are bypassing. In the former case, the ancestor is not itself bypassing but + * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, + * the ancestor's bypass dispatch path must be active even though its own + * bypass_depth remains zero. + * + * This function checks bypass_dsp_enable_depth which is managed separately from + * bypass_depth to enable this decoupling. See enable_bypass_dsp() and + * scx_disable_bypass_dsp(). + */ +static inline bool scx_bypass_dsp_enabled(struct scx_sched *sch) +{ + return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); +} + +/** + * scx_ops_sanitize_err - Sanitize a -errno value + * @sch: scx_sched to error out on error + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.init_task() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static inline int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) +{ + struct scx_sched *root = rcu_dereference_sched(scx_root); + + if (WARN_ON_ONCE(!root)) + return; + + schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); +} /* * Return the rq currently locked from an scx callback, or NULL if no rq is @@ -1573,8 +2201,11 @@ static inline void update_locked_rq(struct rq *rq) /* * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + * + * @ops is the ops table to dispatch through: ops for the cpu form, ops_cid + * for the cid form. */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +#define __SCX_CALL_OP(sch, ops, op, locked_rq, args...) \ do { \ struct rq *__prev_locked_rq; \ \ @@ -1587,6 +2218,9 @@ do { \ update_locked_rq(__prev_locked_rq); \ } while (0) +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ + __SCX_CALL_OP(sch, ops, op, locked_rq, ##args) + #define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ ({ \ struct rq *__prev_locked_rq; \ @@ -1618,17 +2252,36 @@ do { \ * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants * while a previous one is still in progress. */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +#define __SCX_CALL_OP_TASK(sch, ops, op, locked_rq, task, args...) \ do { \ WARN_ON_ONCE(current->scx.kf_tasks[0]); \ current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + __SCX_CALL_OP((sch), ops, op, locked_rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ } while (0) +/* + * A per-task op runs on @task's owner - WARN if @sch isn't it. Sites that must + * target a different scheduler call __SCX_CALL_OP_TASK() directly. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \ + __SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args); \ +} while (0) + +/* + * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs + * this: it takes an arena cmask address instead of a cpumask, so it cannot be + * invoked via its cpu-form set_cpumask() slot. + */ +#define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ + __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) + #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ ({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \ WARN_ON_ONCE(current->scx.kf_tasks[0]); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ @@ -1668,6 +2321,19 @@ static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) } #ifdef CONFIG_EXT_SUB_SCHED +DECLARE_STATIC_KEY_FALSE(__scx_has_subs); + +/** + * scx_has_subs - Whether any sub-scheduler exists + * + * Gates the sub-sched portions of hot paths so that a root-only system doesn't + * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work(). + */ +static inline bool scx_has_subs(void) +{ + return static_branch_unlikely(&__scx_has_subs); +} + /** * scx_task_sched - Find scx_sched scheduling a task * @p: task of interest @@ -1717,14 +2383,20 @@ static inline bool scx_task_on_sched(struct scx_sched *sch, static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { struct sched_ext_ops *ops; - struct scx_sched *root; + struct scx_sched *sch, *root; ops = bpf_prog_get_assoc_struct_ops(aux); - if (likely(ops)) - return rcu_dereference_all(ops->priv); + if (likely(ops)) { + sch = rcu_dereference_all(ops->priv); + if (sch && unlikely(READ_ONCE(sch->dead))) + return NULL; + return sch; + } root = rcu_dereference_all(scx_root); if (root) { + if (unlikely(READ_ONCE(root->dead))) + return NULL; /* * COMPAT-v6.19: Schedulers built before sub-sched support was * introduced may have unassociated non-struct_ops programs. @@ -1755,7 +2427,10 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) else return NULL; } + #else /* CONFIG_EXT_SUB_SCHED */ +static inline bool scx_has_subs(void) { return false; } + static inline struct scx_sched *scx_task_sched(const struct task_struct *p) { return rcu_dereference_protected(scx_root, @@ -1776,10 +2451,15 @@ static inline bool scx_task_on_sched(struct scx_sched *sch, static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { - return rcu_dereference_all(scx_root); + struct scx_sched *root = rcu_dereference_all(scx_root); + + if (root && unlikely(READ_ONCE(root->dead))) + return NULL; + return root; } static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } + #endif /* CONFIG_EXT_SUB_SCHED */ #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c new file mode 100644 index 000000000000..0554448835bd --- /dev/null +++ b/kernel/sched/ext/sub.c @@ -0,0 +1,2681 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * A sub-scheduler is an scx_sched attached to a cgroup subtree under another + * scx_sched. This file holds the sub-scheduler implementation: the scheduler + * tree walk, capability delegation, per-shard cap state and its sync, and the + * sub-scheduler enable/disable paths. The core dispatch/enqueue machinery it + * builds on lives in ext.c. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "sub.h" +#include "inlines.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +/* + * On while any sub-scheduler exists so that a root-only system doesn't pay for + * the sub-sched portions of hot paths. See scx_has_subs(). + */ +DEFINE_STATIC_KEY_FALSE(__scx_has_subs); + +/* latched at root enable before any rescue runs */ +static s32 scx_rescue_bw_1024; +static s64 scx_rescue_quantum_ns; +static s64 scx_rescue_sat_delta_ns; +static unsigned long scx_rescue_decay_halflife; +static unsigned long scx_rescue_overload_after; + +/** + * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk + * @pos: current position + * @root: walk root + * + * In a walk started by scx_next_descendant_pre(), continue past @pos's subtree: + * return @pos's next sibling, or the closest ancestor's next sibling, or NULL + * if @pos's subtree is the last under @root. Same locking rules. + */ +struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock) || + rcu_read_lock_any_held()); + + while (pos != root) { + next = list_next_or_null_rcu(&scx_parent(pos)->children, &pos->sibling, + struct scx_sched, sibling); + if (next) + return next; + pos = scx_parent(pos); + } + return NULL; +} + +/** + * scx_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: sched whose descendants to walk + * + * To be used by scx_for_each_descendant_pre(). Find the next descendant to + * visit for pre-order traversal of @root's descendants. @root is included in + * the iteration and the first node to be visited. + */ +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock) || + rcu_read_lock_any_held()); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = list_first_or_null_rcu(&pos->children, struct scx_sched, sibling); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + return scx_skip_subtree_pre(pos, root); +} + +static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) +{ + return rhashtable_lookup(&scx_sched_hash, &cgroup_id, + scx_sched_hash_params); +} + +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) +{ + rcu_assign_pointer(p->scx.sched, sch); +} + +struct cgroup *sch_cgroup(struct scx_sched *sch) +{ + return sch->cgrp; +} + +/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) +{ + struct cgroup *pos; + struct cgroup_subsys_state *css; + + cgroup_for_each_live_descendant_pre(pos, css, cgrp) + rcu_assign_pointer(pos->scx_sched, sch); +} + +static void free_pshard(struct scx_pshard *pshard) +{ + struct scx_caps_updated *cu; + + if (!pshard) + return; + cu = &pshard->caps_updated; + if (cu->cmask_arena_out) + scx_arena_free(pshard->sch, cu->cmask_arena_out, + struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(pshard->nr_cids))); + kfree(pshard); +} + +void scx_free_pshards(struct scx_sched *sch) +{ + s32 si; + + if (!sch->pshard) + return; + for (si = 0; si < sch->nr_pshards; si++) + free_pshard(sch->pshard[si]); + kfree(sch->pshard); +} + +static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node) +{ + const struct scx_cid_shard *shard = + &rcu_dereference_protected(scx_cid_shard_ranges, + lockdep_is_held(&scx_enable_mutex))[shard_idx]; + size_t cmask_size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(shard->nr_cids)); + struct scx_pshard *pshard; + struct scx_caps_updated *cu; + s32 i; + + pshard = kzalloc_node(sizeof(*pshard), GFP_KERNEL, node); + if (!pshard) + return NULL; + + raw_spin_lock_init(&pshard->lock); + pshard->sch = sch; + pshard->base = shard->base_cid; + pshard->nr_cids = shard->nr_cids; + + for (i = 0; i < __SCX_NR_CAPS; i++) + scx_cmask_init(&pshard->caps[i].cmask, shard->base_cid, shard->nr_cids); + + cu = &pshard->caps_updated; + raw_spin_lock_init(&cu->lock); + INIT_LIST_HEAD(&cu->node_in_flight); + __scx_cmask_init(&cu->cmask, shard->base_cid, shard->nr_cids, SCX_CID_SHARD_MAX_CPUS); + + cu->cmask_arena_out = scx_arena_alloc(sch, cmask_size); + if (!cu->cmask_arena_out) { + free_pshard(pshard); + return NULL; + } + + scx_cmask_init(cu->cmask_arena_out, shard->base_cid, shard->nr_cids); + + return pshard; +} + +s32 scx_alloc_pshards(struct scx_sched *sch) +{ + struct scx_pshard **pshard; + s32 *shard_node; + s32 si; + + if (!sch->is_cid_type || !sch->arena_pool) + return 0; + + shard_node = rcu_dereference_protected(scx_shard_node, + lockdep_is_held(&scx_enable_mutex)); + + pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL); + if (!pshard) + return -ENOMEM; + + for (si = 0; si < scx_nr_cid_shards; si++) { + pshard[si] = alloc_pshard(sch, si, shard_node[si]); + if (!pshard[si]) { + while (--si >= 0) + free_pshard(pshard[si]); + kfree(pshard); + return -ENOMEM; + } + } + + sch->nr_pshards = scx_nr_cid_shards; + /* + * Publish only after every entry is built so a reader observing + * @sch->pshard never sees a partially-filled array or unpublished cid + * tables. Pair the store with a barrier and an acquire load on the + * read side. + */ + smp_wmb(); + WRITE_ONCE(sch->pshard, pshard); + return 0; +} + +/* + * Seed the root's caps fully. Root owns all cids on all caps at enable time. + * Children acquire caps via scx_bpf_sub_grant(). + */ +void scx_init_root_caps(struct scx_sched *sch) +{ + s32 si, i; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + for (i = 0; i < __SCX_NR_CAPS; i++) + scx_cmask_fill(&ps->caps[i].cmask); + } +} + +/* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */ +static s64 scx_rescue_slice_remaining(struct rq *rq) +{ + s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap; + + return max(rq->scx.rescue.slice - served, 0); +} + +/* + * Decay @pcpu's rescue usage average in place, halving per the knob-derived + * halflife, see scx_rescue_set_knobs(). The timestamp advances only by whole + * halflives. + */ +static u64 scx_rescue_decay_avg(struct scx_sched_pcpu *pcpu) +{ + unsigned long halflife = scx_rescue_decay_halflife; + u64 n = div_u64(get_jiffies_64() - pcpu->rescue_avg_at, halflife); + + if (n) { + pcpu->rescue_avg = n < 64 ? pcpu->rescue_avg >> n : 0; + pcpu->rescue_avg_at += n * halflife; + } + return pcpu->rescue_avg; +} + +/** + * scx_rescue_charge - Charge the rescuee's runtime + * @rq: rq the rescuee is running on + * @delta_exec: runtime being charged + * + * Also ends the rescue once the admitted slice has been served in full. Ending + * on served time rather than slice exhaustion bounds both the rescue and the + * charging when a scheduler extends the rescuee's slice. + */ +void scx_rescue_charge(struct rq *rq, s64 delta_exec) +{ + struct scx_sched_pcpu *pcpu; + + lockdep_assert_rq_held(rq); + + /* + * A rescue slice is bounded by one quantum and tick-driven expiry can + * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs. + */ + delta_exec = min_t(s64, delta_exec, scx_rescue_quantum_ns + TICK_NSEC); + + rq->scx.rescue.budget -= delta_exec; + + /* per-cpu usage average feeds the overload victim pick */ + pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; + + if (!scx_rescue_slice_remaining(rq)) + scx_task_slice_ended(rq, rq->scx.rescue.curr); +} + +/** + * scx_rescue_end - End the rescue execution on @rq + * @rq: rq of interest + * + * When no rescuee is left pending, the session is over and the balance above + * one quantum dies with it - it would otherwise become a banked license to + * preempt the cid owner long after the starvation ended. While waiters remain, + * the accrued deficit belongs to the queue and carries into the next rescue. + */ +void scx_rescue_end(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + rq->scx.rescue.curr = NULL; + if (list_empty(&rq->scx.rescue.dsq.list)) + rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns); +} + +/** + * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee + * @rq: rq @p is running on + * @p: task under rescue whose slice is exhausted + * + * Called from put_prev_task_scx() to decide what an exhausted slice means for + * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted + * slice is fully served, so arriving here with the rescue still open means @p + * was preempted. Restore the unserved remainder and return %true - @p stays the + * rescuee and the caller reinserts it at the tail of the local DSQ, behind + * whatever preempted the rescuee. + * + * Return %false to end the rescue instead - the slice is already fully served, + * @p is leaving the rq or bypass is dismantling rescues. + */ +bool scx_rescue_keep(struct rq *rq, struct task_struct *p) +{ + s64 remaining = scx_rescue_slice_remaining(rq); + + lockdep_assert_rq_held(rq); + + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || + scx_bypassing(scx_task_sched(p), cpu_of(rq))) + return false; + + scx_set_task_slice(p, remaining); + return true; +} + +/** + * scx_rescue_accrue - Accrue budget at the configured fraction of elapsed time + * @rq: rq of interest + * + * A session spans from the first arrival until no rescuee is left, pending or + * admitted. While one is active the cap is three quanta and the balance drives + * escalation, see scx_rescue_timerfn(). Outside a session the cap is one + * quantum, so an idle gap funds the next arrival's admission but never an + * escalation. + */ +static void scx_rescue_accrue(struct rq *rq) +{ + bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list); + s64 cap = in_session ? 3 * scx_rescue_quantum_ns : scx_rescue_quantum_ns; + s64 delta; + u64 now; + + lockdep_assert_rq_held(rq); + + /* not every path here holds an updated rq clock, use __scx_bpf_now() */ + now = __scx_bpf_now(rq); + delta = now - rq->scx.rescue.clock; + rq->scx.rescue.clock = now; + + /* + * Avoid multiplication overflows by taking a shortcut when the gap is + * large enough to fill the budget. + */ + if (delta >= scx_rescue_sat_delta_ns) + rq->scx.rescue.budget = cap; + else + rq->scx.rescue.budget = + min(cap, rq->scx.rescue.budget + + ((delta * scx_rescue_bw_1024) >> SCHED_CAPACITY_SHIFT)); +} + +/* + * The slice for the next admission - the quantum divided across the stranded + * tasks so that a crowded queue round-robins on shorter slices. + */ +static s64 scx_rescue_next_slice(struct rq *rq) +{ + s64 min_slice = max_t(s64, SCX_RESCUE_MIN_SLICE_US * NSEC_PER_USEC, TICK_NSEC); + u32 depth = rq->scx.rescue.dsq.nr ?: 1; + + return clamp(div_s64(scx_rescue_quantum_ns, depth), min_slice, scx_rescue_quantum_ns); +} + +static void scx_rescue_timer_arm(struct rq *rq) +{ + struct timer_list *timer = &rq->scx.rescue.timer; + s64 delay = scx_rescue_quantum_ns / 4; /* should be granular enough */ + + if (timer_pending(timer)) + return; + + /* + * While the head waiter can't be admitted because the bucket is short + * of a full quantum, stretch to the full funding delay. + */ + if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) { + s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget; + + delay = max(delay, + div_s64(deficit << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024)); + } + + /* +1 rounds up so the beat is due by the time the timer fires */ + timer->expires = jiffies + nsecs_to_jiffies(delay) + 1; + add_timer_on(timer, cpu_of(rq)); +} + +/** + * scx_rescue_admit - Start rescuing @p on @rq + * @rq: rq @p is being admitted on + * @p: task being admitted, off any DSQ + * @slice: CPU time to grant + * + * The schedulers keep their normal control over @p and may preempt or reslice + * it. @slice is measured on served CPU time against the snapshot taken here, so + * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep(). + * Prolonged denial escalates into protected execution, see + * scx_rescue_timerfn(). + */ +static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(rq->scx.rescue.curr); + + rq->scx.rescue.curr = p; + rq->scx.rescue.slice = slice; + rq->scx.rescue.exec_snap = p->se.sum_exec_runtime; + scx_set_task_slice(p, slice); + scx_rescue_timer_arm(rq); +} + +/** + * scx_rescue_try_admit - Try to admit a freshly stranded task + * @rq: rq @p is being inserted on + * @p: stranded task being diverted to rescue + * + * One rescue at a time and earlier arrivals go first. Admission needs a full + * quantum of budget, spent as the rescue runs. Return %true if @p was admitted + * and should be inserted at the tail of @rq's local DSQ, %false if it has to + * park on the rescue DSQ, with the timer armed to admit it later. + */ +static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) +{ + scx_rescue_accrue(rq); + + if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) && + rq->scx.rescue.budget >= scx_rescue_quantum_ns) { + scx_rescue_admit(rq, p, scx_rescue_quantum_ns); + return true; + } + + scx_rescue_timer_arm(rq); + return false; +} + +/** + * scx_rescue_check_overload - Eject the top rescue consumer on a stuck rescue + * @rq: rq whose rescue timer fired + * + * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue + * demand on this cpu persistently exceeds the configured bandwidth. Eject the + * sub with the highest recent rescue consumption instead of letting the + * scheduler stall path blame the waiter's owner, who may just be crowded out. + */ +static void scx_rescue_check_overload(struct rq *rq) +{ + struct scx_sched *victim = NULL, *pos; + struct task_struct *p; + int cpu = cpu_of(rq); + u64 max_avg = 0; + u32 dur_ms; + + lockdep_assert_rq_held(rq); + + p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + if (!p) + return; + + /* has the head waiter been queued for longer than the threshold? */ + if (time_before(jiffies, p->scx.rescue_at + scx_rescue_overload_after)) + return; + + /* + * Grace period after the last ejection on this cpu - the freed + * bandwidth gets one threshold's worth of time to drain the backlog + * before another sub is judged. + */ + if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at + + scx_rescue_overload_after)) + return; + + list_for_each_entry_rcu(pos, &scx_sched_all, all) { + u64 avg = scx_rescue_decay_avg(per_cpu_ptr(pos->pcpu, cpu)); + + /* skip an already-exiting sub, else the ejection is wasted */ + if (pos->level && avg > max_avg && + atomic_read(&pos->exit_kind) == SCX_EXIT_NONE) { + max_avg = avg; + victim = pos; + } + } + if (!victim) + return; + + rq->scx.rescue.kill_at = get_jiffies_64(); + dur_ms = jiffies_to_msecs(jiffies - p->scx.rescue_at); + __scx_exit(victim, SCX_EXIT_ERROR_RESCUE, 0, cpu, + "used too much rescue CPU time (%llums) while %s[%d] waited %u.%03us to be rescued", + div_u64(max_avg, NSEC_PER_MSEC), p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); +} + +/** + * scx_rescue_timerfn - Drive and pace rescue execution + * @timer: rq->scx.rescue.timer + * + * Runs every quarter quantum while a rescuee exists, pending or admitted, see + * scx_rescue_timer_arm(). The head waiter is admitted once the bucket holds a + * full quantum and granted its slice, see scx_rescue_next_slice(). A session + * whose budget accumulates over two quanta with the admitted rescuee still + * waiting escalates - the rescuee's remaining slice turns into protected + * execution and it preempts the current task. An overloaded rescue queue ejects + * the top consumer, see scx_rescue_check_overload(). + */ +static void scx_rescue_timerfn(struct timer_list *timer) +{ + struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + + p = rq->scx.rescue.curr; + if (!p && list_empty(&rq->scx.rescue.dsq.list)) + return; + + scx_rescue_accrue(rq); + scx_rescue_check_overload(rq); + + if (!p) { + s64 slice = scx_rescue_next_slice(rq); + + /* no rescue in progress */ + if (rq->scx.rescue.budget < scx_rescue_quantum_ns) + goto out_arm; + + /* there's enough budget to start rescuing the next one */ + p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_rescue_admit(rq, p, slice); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + resched_curr(rq); + } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { + /* + * The rescuee waited for the CPU for too long. Escalate - grant + * the unserved remainder, protect it from the schedulers and + * preempt the current task. The slice is set before the + * protection. Repeat beats only repeat the head move - the + * slice write is refused on a protected task. + */ + scx_set_task_slice(p, scx_rescue_slice_remaining(rq)); + p->scx.flags |= SCX_TASK_PROTECTED; + scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, + &rq->scx.local_dsq, rq); + } +out_arm: + scx_rescue_timer_arm(rq); +} + +/* flush out tasks waiting for rescue before a CPU goes down */ +void scx_rescue_flush(struct rq *rq) +{ + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + /* sched domain rebuilds call rq_offline with the CPU staying alive */ + if (cpu_active(cpu_of(rq))) + return; + + /* end the current rescue */ + if (rq->scx.rescue.curr) + scx_task_slice_ended(rq, rq->scx.rescue.curr); + + /* and flush out all pending ones */ + list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + } + + timer_delete(&rq->scx.rescue.timer); +} + +void scx_rescue_dump(struct seq_buf *s, struct rq *rq) +{ + struct task_struct *p = rq->scx.rescue.curr; + + scx_dump_line(s, " rescue=%u budget=%lldus rescuing=%s[%d]", + rq->scx.rescue.dsq.nr, + div_s64(rq->scx.rescue.budget, NSEC_PER_USEC), + p ? p->comm : "none", p ? p->pid : -1); +} + +/* + * A scheduler whose stall watchdog is shorter than the overload threshold gets + * stall-killed over its parked waiters before the overload check can eject the + * actual top consumer. The root's knobs set the threshold, warn on any + * scheduler that doesn't fit it. + */ +static void scx_rescue_check_timeout(struct scx_sched *sch) +{ + if (!scx_rescue_bw_1024 || sch->watchdog_timeout > scx_rescue_overload_after) + return; + + pr_warn("sched_ext: %s: watchdog timeout %ums <= rescue overload threshold %ums\n", + sch->ops.name, jiffies_to_msecs(sch->watchdog_timeout), + jiffies_to_msecs(scx_rescue_overload_after)); +} + +/* latch the rescue parameters on root scheduler enable */ +void scx_rescue_set_knobs(struct scx_sched *sch) +{ + s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT; + s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US; + s64 period_ns; + + if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) { + scx_rescue_bw_1024 = 0; + return; + } + + scx_rescue_bw_1024 = bw_ppt * SCHED_CAPACITY_SCALE / 1000; + scx_rescue_quantum_ns = max(quantum_us * NSEC_PER_USEC, TICK_NSEC); + scx_rescue_sat_delta_ns = + div_s64((4 * scx_rescue_quantum_ns + TICK_NSEC) << SCHED_CAPACITY_SHIFT, + scx_rescue_bw_1024); + + /* + * The overload threshold and the decay halflife scale with the funding + * period - the time the bucket takes to fund one full quantum. + */ + period_ns = div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024); + scx_rescue_overload_after = + clamp(nsecs_to_jiffies(SCX_RESCUE_OVERLOAD_MULT * period_ns), + msecs_to_jiffies(SCX_RESCUE_MIN_OVERLOAD_MS), + msecs_to_jiffies(SCX_RESCUE_MAX_OVERLOAD_MS)); + scx_rescue_decay_halflife = scx_rescue_overload_after / 4; + + /* a single in-budget wait must not cross the overload trigger */ + if (nsecs_to_jiffies(period_ns) > scx_rescue_overload_after / 2) + pr_warn("sched_ext: %s: rescue funding period %lldms > overload threshold %ums / 2\n", + sch->ops.name, div_s64(period_ns, NSEC_PER_MSEC), + jiffies_to_msecs(scx_rescue_overload_after)); + + scx_rescue_check_timeout(sch); +} + +void scx_rescue_init(struct rq *rq) +{ + BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL)); + timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED); + rq->scx.rescue.kill_at = get_jiffies_64(); +} + +/** + * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * @sch: enqueuing sub-sched + * @rq: rq whose local DSQ @p targets + * @p: task being inserted + * @enq_flags: in/out, unhonored flags are cleared + * + * Return @rq's local DSQ if @sch holds the required caps on @rq's cid. + * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and + * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on + * @p. + * + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting + * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback + * migration-disabled admission. + * + * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued + * to and run by its nearest non-bypassing ancestor. If root is bypassing, it + * always holds all caps. + */ +struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; + u64 needed = scx_caps_for_enq(*enq_flags); + u64 missing; + + /* + * On a remote activation the scheduling sched (@asch) differs from + * @p's owner (@sch). Check caps against the scheduling sched. + */ + if (*enq_flags & SCX_ENQ_PREEMPT) + needed |= scx_caps_for_preempt(asch, rq, *enq_flags); + missing = scx_missing_caps(asch, cpu_of(rq), needed); + + /* requirements met */ + if (likely(!missing)) + return &rq->scx.local_dsq; + + /* + * The task must run on this CPU regardless of caps: the rq is draining + * offline (BPF scheduler bypassed), the task is migration-disabled, or a + * migration is pending. Admit despite the missing caps and count it. + * Refuse preemptions. + */ + if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || + p->migration_pending)) { + __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); + *enq_flags &= ~SCX_ENQ_PREEMPT; + return &rq->scx.local_dsq; + } + + /* + * Diverting to rescue or reject, neither of which honors IMMED, PREEMPT + * or HEAD - a diversion has no priority and IMMED is not allowed on + * non-local DSQs. Strip the enq and task flags along with the slice. + */ + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + p->scx.flags &= ~SCX_TASK_IMMED; + + /* the enqueuer opted for rescue instead of rejection and reenqueue */ + if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); + if (scx_rescue_try_admit(rq, p)) + return &rq->scx.local_dsq; + + /* queueing, the overload trigger measures the wait from here */ + p->scx.rescue_at = jiffies; + return &rq->scx.rescue.dsq; + } + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = cid; + + return &rq->scx.reject_dsq; +} + +/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) +{ + u64 missing; + + /* migration-disabled tasks and the rescuee are admitted capless */ + if (is_migration_disabled(p) || p == scx_rescuee(rq)) + return false; + + missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + if (likely(!missing)) + return false; + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + return true; +} + +/* + * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides + * from p->scx.reenq_reason_*. + * + * A task can be re-rejected repeatedly. The reenqueue is bounded per task in + * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. + * Rejection can't happen for root. + */ +void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move to a private list so a task re-rejected by the + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + /* migration_pending tasks should have bypassed to local DSQ */ + if (WARN_ON_ONCE(p->migration_pending)) + continue; + + scx_dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= SCX_TASK_REENQ_CAP; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } +} + +/* record a caps change, see struct scx_caps_updated */ +static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, + struct list_head *to_deliver) +{ + struct scx_caps_updated *cu = &ps->caps_updated; + + guard(raw_spinlock)(&cu->lock); + scx_cmask_or(&cu->cmask, cids); + cu->caps |= caps; + if (list_empty(&cu->node_in_flight)) + list_add_tail(&cu->node_in_flight, to_deliver); +} + +/* deliver queued caps_updated callbacks, see struct scx_caps_updated */ +static void caps_updated_deliver(struct list_head *to_deliver) +{ + struct scx_caps_updated *cu, *tmp; + + list_for_each_entry_safe(cu, tmp, to_deliver, node_in_flight) { + struct scx_pshard *ps = container_of(cu, struct scx_pshard, caps_updated); + struct scx_sched *sch = ps->sch; + + while (true) { + u64 caps = 0; + + /* + * During enable, has_op is set after ops.sub_attach(), + * so !has_op means the op is absent or the sched isn't + * live yet - e.g. caps grant from ops.sub_attach(). + * Either way don't consume - leave for + * scx_sub_seed_caps() to deliver once live. + */ + scoped_guard (raw_spinlock, &cu->lock) { + if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) && + likely(!READ_ONCE(sch->aborting))) { + struct scx_cmask_ref ref; + + caps = cu->caps; + scx_cmask_ref_init_kern(sch, cu->cmask_arena_out, + ps->base, ps->nr_cids, &ref); + scx_cmask_ref_copy(&ref, &cu->cmask); + scx_cmask_clear(&cu->cmask); + cu->caps = 0; + } else { + list_del_init(&cu->node_in_flight); + } + } + if (!caps) + break; + + /* caps != 0 only when deliverable (has_op, above) */ + SCX_CALL_OP(sch, sub_caps_updated, NULL, cu->cmask_arena_out, caps); + } + } +} + +/* + * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant + * taken during its sub_attach(), before has_op was set). Called once @sch is + * enabled. + */ +static void scx_sub_seed_caps(struct scx_sched *sch) +{ + LIST_HEAD(to_deliver); + s32 si; + + guard(irqsave)(); + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + struct scx_caps_updated *cu = &ps->caps_updated; + + scoped_guard (raw_spinlock, &cu->lock) { + if (cu->caps && list_empty(&cu->node_in_flight)) + list_add_tail(&cu->node_in_flight, &to_deliver); + } + } + caps_updated_deliver(&to_deliver); +} + +static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) +{ + u64 ecaps = 0; + u32 cap_bit; + + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + if (scx_cmask_test(cid, &ps->caps[cap_bit].cmask)) + ecaps |= BIT_U64(cap_bit) | scx_caps_implied(BIT_U64(cap_bit)); + return ecaps; +} + +/** + * queue_sync_ecaps - Queue ecaps update for a (sch, cid) pair + * @sch: sched to update + * @cid: cid to update + * + * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in + * dispatch_one(). + */ +static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) +{ + s32 cpu = __scx_cid_to_cpu(cid); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * below sees the node off the list and queues it, or the in-flight sync + * sees the caps[] update made before this call. + */ + smp_mb(); + + /* @cid's pshard->lock excludes concurrent queueing attempts */ + if (llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) + scx_kick_cpu(sch->ancestors[0], cpu, 0); +} + +/* discard @rq's queued ecaps syncs */ +static void discard_queued_syncs(struct rq *rq) +{ + struct llist_node *pos, *tmp; + + lockdep_assert_rq_held(rq); + + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) + init_llist_node(pos); +} + +/** + * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * @rq: the cid's cpu rq + * @prev: @rq's previous task from the in-progress dispatch + * + * pshard->caps[] is the target configuration. pcpu->ecaps is the effective + * transposed copy owned by the cid's cpu and written only here under @rq's + * lock. + * + * A sched that newly gains baseline access here is owed an update_idle() so it + * learns the cid's idle state. Such a gain arms the per-rq + * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. + */ +void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + s32 cpu = cpu_of(rq); + s32 cid, shard; + struct llist_node *batch, *pos, *tmp; + u64 lost_all = 0; + + lockdep_assert_rq_held(rq); + + if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + return; + + /* + * ecaps are zeroed while the cpu is inactive and must stay zero. + * Discard queued syncs instead of processing them - the + * scx_online_ecaps() reseed re-syncs every sched on activation. + * cpu_active() clears before the offline zeroing and sets before the + * reseed is queued, so this test can neither miss a racing sync nor + * eat the reseed. + */ + if (unlikely(!cpu_active(cpu))) { + discard_queued_syncs(rq); + return; + } + + /* @cid is valid here: the cpu is active with queued syncs */ + cid = __scx_cpu_to_cid(cpu); + shard = rcu_dereference_all(scx_cid_to_shard)[cid]; + + batch = llist_del_all(&rq->scx.ecaps_to_sync); + llist_for_each_safe(pos, tmp, batch) { + struct scx_sched_pcpu *pcpu = + container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); + struct scx_pshard *ps = pcpu->sch->pshard[shard]; + u64 old, ecaps, lost, gained; + + init_llist_node(pos); + + /* pairs with smp_mb() in queue_sync_ecaps(), see there */ + smp_mb(); + + old = READ_ONCE(pcpu->ecaps); + ecaps = calc_effective_caps(ps, cid); + WRITE_ONCE(pcpu->ecaps, ecaps); + + lost = old & ~ecaps; + gained = ecaps & ~old; + lost_all |= lost; + + /* + * Tell the sched its effective caps on this cid changed. The + * invocation is equivalent to the dispatch path and may drop + * and re-acquire the rq lock temporarily while the rest of + * @batch is held privately, see scx_discard_ecaps_to_sync(). + * The dispatch kfuncs resolve their context on the executing + * cpu, which under core scheduling can differ from @rq's cpu, + * so the context is set up there. The rq recorded in it keeps + * the dispatches targeting @rq. + */ + if (ecaps != pcpu->reported_ecaps && + SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && + !scx_bypassing(pcpu->sch, cpu)) { + struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + + dspc->rq = rq; + /* stash @prev so nested dispatches can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + rq->scx.sub_dispatch_prev = NULL; + scx_flush_dispatch_buf(pcpu->sch, rq); + pcpu->reported_ecaps = ecaps; + } + + /* + * Gaining baseline access owes an update_idle() so the sched + * learns the cpu's idle state. Arm the per-rq gate so the next + * idle pick flushes it. Losing access drops any pending notify. + */ + if (gained & SCX_CAP_BASE) { + pcpu->idle_renotify = true; + rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; + } else if (lost & SCX_CAP_BASE) { + pcpu->idle_renotify = false; + } + } + + /* + * Losing a cap can strand already-queued tasks. Schedule a reenq scan + * to move the now-capless ones off the local DSQ. The scan tests + * against the effective caps and thus must come after the ecaps sync. + */ + if (lost_all & SCX_CAPS_REENQ_ON_LOSS) + scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); +} + +/** + * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification + * @rq: rq of the cpu leaving bypass + * @sch: scheduler that just left bypass on @rq's cpu + * + * scx_process_sync_ecaps() consumes syncs while bypassing without delivering + * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a + * sync when bypass lifts, so without a replay a cid that never changes again + * would never be notified. The attach-time initial grants are the acute case + * as they are consumed during the enable bypass window. Re-queue a sync for + * any undelivered delta so the next dispatch delivers it. + */ +void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) +{ + s32 cpu = cpu_of(rq); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + struct scx_pshard *ps; + s32 cid; + + lockdep_assert_rq_held(rq); + + /* root holds every cap and never uses ecaps */ + if (!sch->level) + return; + + if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps) + return; + + cid = __scx_cpu_to_cid(cpu); + ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; + + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(sch, cid); +} + +/* + * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync + * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() + * only on a real change since offline. + */ +void scx_online_ecaps(struct rq *rq) +{ + struct scx_sched *root, *pos; + s32 cid, shard; + + /* + * Only a live hierarchy can have ecaps to reseed. This also keeps the + * table reads below away from an enable that failed before publishing + * the tables. A concurrent disable can't retire them, see + * handle_hotplug(). + */ + if (!scx_enabled()) + return; + + guard(rq_lock_irqsave)(rq); + + root = scx_root_protected(); + cid = __scx_cpu_to_cid(cpu_of(rq)); + shard = rcu_dereference_all(scx_cid_to_shard)[cid]; + + scx_for_each_descendant_pre(pos, root) { + struct scx_pshard *ps; + + /* root holds every cap and never uses ecaps */ + if (!pos->level) + continue; + + ps = pos->pshard[shard]; + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(pos, cid); + } +} + +/* + * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks + * treat the cpu as capless while offline. Pending and late-queued syncs are + * discarded at consumption by scx_process_sync_ecaps() while the cpu is + * inactive. Leave reported_ecaps. Ownership is unchanged, so the + * scx_online_ecaps() reseed reports only a genuine delta. No callback fires + * here. + */ +void scx_offline_ecaps(struct rq *rq) +{ + s32 cpu = cpu_of(rq); + struct scx_sched *root, *pos; + + guard(rq_lock_irqsave)(rq); + + root = scx_root_protected(); + + scx_for_each_descendant_pre(pos, root) { + /* root holds every cap and never uses ecaps */ + if (!pos->level) + continue; + + WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); + } +} + +/* + * @pcpu's sched was unhashed before the grace period, so nothing re-queues its + * sync node. Remove the node from @rq's pending list so the pcpu can be freed. + */ +void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) +{ + struct rq *rq = cpu_rq(cpu); + struct llist_node *head = NULL, *tail = NULL; + struct llist_node *pos, *tmp; + + /* + * llist can't unlink a single node. Take all queued nodes, drop @pcpu's + * and resplice the rest. Nodes in the taken batch read as on-list + * throughout, so queue_sync_ecaps() stays correct. + */ + if (llist_on_list(&pcpu->ecaps_to_sync_node)) { + scoped_guard (rq_lock_irqsave, rq) { + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { + if (pos == &pcpu->ecaps_to_sync_node) { + init_llist_node(pos); + } else { + pos->next = head; + head = pos; + if (!tail) + tail = pos; + } + } + if (head) + llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); + } + } + + /* + * An in-flight scx_process_sync_ecaps() batch may still hold the node + * privately across dispatch-induced rq unlocks, reading as on-list. + * + * Because a bypassing sched gets no op call, init_llist_node() and all + * @pcpu accesses share one contiguous lock hold, off-list under the rq + * lock means @pcpu won't be accessed again. + */ + while (true) { + scoped_guard (rq_lock_irqsave, rq) { + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + } + cpu_relax(); + } +} + +/** + * scx_discard_stale_ecaps_syncs - Discard ecaps syncs from earlier schedulers + * + * To be called during root enable before the scheduler goes live. An earlier + * root's sub-sched may not have gone through its RCU free path yet (e.g. a + * still-open link fd defers it) and can leave queued ecaps syncs behind. + * Processing them would decode the dead sched's pshards with the current cid + * layout. Discard them instead. The backing scx_sched_pcpu's are still + * allocated as the free path removes ecaps_to_sync_node before freeing. + */ +void scx_discard_stale_ecaps_syncs(void) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + discard_queued_syncs(rq); + } +} + +static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); + +void drain_descendants(struct scx_sched *sch) +{ + /* + * Child scheds that finished the critical part of disabling will take + * themselves off @sch->children. Wait for it to drain. As propagation + * is recursive, empty @sch->children means that all proper descendant + * scheds reached unlinking stage. + */ + wait_event(scx_unlink_waitq, list_empty(&sch->children)); +} + +/** + * scx_rehome_task - Move a task to a sched it has been initialized for + * @to: sched taking over @p, @p's init on it already complete + * @p: task to re-home + * + * Exit @p from its current sched and switch it over to @to, overriding the + * state to %SCX_TASK_READY to account for the already completed init. A task + * on a non-ext class, possible under an %SCX_OPS_SWITCH_PARTIAL root, stays + * %READY and is enabled by switching_to_scx() if it switches over. + */ +static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, to); + scx_set_task_state(p, SCX_TASK_READY); + if (p->sched_class == &ext_sched_class) + scx_enable_task(to, p); + } +} + +/** + * scx_punt_task - Hand a task to a failed sched without initialization + * @to: failed and bypassed sched taking custody of @p + * @p: task to punt + * + * Take @p off its current sched and put it on @to at %SCX_TASK_NONE. @to is + * dying and its teardown will re-home @p properly. + * + * Used when @to must take over @p but failed to initialize it. Bypass keeps + * scheduling decisions away from @to but @p can still trigger its task ops, + * which may confuse the BPF side. @to is dying anyway. The exit paths skip + * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). + */ +static void scx_punt_task(struct scx_sched *to, struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_sched(p, to); + } +} + +static void scx_fail_parent(struct scx_sched *sch, + struct task_struct *failed, s32 fail_code) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + + scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", + fail_code, failed->comm, failed->pid); + + /* + * Once $parent is bypassed, tasks can be punted into it. This may + * cause downstream failures on the BPF side but $parent is dying + * anyway. + */ + scx_bypass(parent, true); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (scx_task_on_sched(parent, p)) + continue; + + scx_punt_task(parent, p); + } + scx_task_iter_stop(&sti); +} + +#ifdef CONFIG_EXT_GROUP_SCHED +/** + * scx_cgroup_claim_subtree - Claim the subtree's cgroups for an enabling sub + * @sch: sub-scheduler being enabled + * + * Called while enabling @sch, after the subtree's cgrp->scx_sched's are pointed + * at @sch and before any task is claimed. This mirrors root enable's + * cgroups-before-tasks order. The ops.init_task() args are task_group-granular + * and can still reference a cgroup outside the handed-over set when the cpu + * controller is coarser than the sub topology or mounted on cgroup1. + * + * First init each of the parent sched's subtree cgroups on @sch, and only then + * exit them from the parent, so that a failed init can be unwound with the + * parent untouched. The both-inited transient is invisible outside + * scx_cgroup_lock(). %SCX_TG_SUB_INIT tracks the first pass's progress. + * %SCX_TG_INITED stays set throughout, except for a task_group whose + * ops.cgroup_init() failed on the parent (see scx_cgroup_return_subtree()): + * there is nothing to exit from the parent and %SCX_TG_INITED is set back with + * the transfer. + * + * Dying but not yet offlined task_groups are included: a removed cgroup keeps + * hosting scheduling events until its dying tasks finish their final context + * switches, so it still needs to be inited on a sched, and its offline-time + * ops.cgroup_exit() follows the last of those events. + * + * Return 0 on success, -errno on failure. On failure, @sch has been + * scx_error()'d and is left with no cgroups. + */ +static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + int ret; + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + if (tg->scx.sched != parent || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + if (SCX_HAS_OP(sch, cgroup_init)) { + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + goto err; + } + } + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + /* + * SUB_INIT is pass 1's progress mark: pass 2 and the err path + * must visit exactly the tgs pass 1 inited. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + /* skip the exit if the parent's ops.cgroup_init() failed */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(parent, cgroup_exit)) + SCX_CALL_OP(parent, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = sch; + tg->scx.flags |= SCX_TG_INITED; + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + + return 0; + +err: + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + if (SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + return ret; +} + +/** + * scx_cgroup_return_subtree - Return the subtree's cgroups to the parent sched + * @sch: sub-scheduler being disabled + * + * Called while disabling @sch, after the subtree's cgrp->scx_sched's are reset + * to the parent sched and before tasks are re-homed, mirroring root disable's + * cgroups-before-tasks teardown order. The reverse of + * scx_cgroup_claim_subtree(): exit @sch's cgroups from @sch, then init them on + * the parent with the current tg->scx.* values, resyncing settings that changed + * while @sch had them. + * + * When an init on the parent fails, the parent is failed - the same policy as + * task re-homing. The remaining task_groups are punted: they move to the parent + * anyway with %SCX_TG_INITED cleared, as ops.cgroup_init() failed or never ran + * for them. A punted task_group gets no cgroup ops. The dying parent's own + * disable moves it one sched up, initing it there. Root ends the chain: root + * teardown drops cgroup ops entirely and the next enable's bulk init re-inits + * every online task_group. + * + * The task re-home that follows still delivers ops.init_task() to the dying + * parent, including for tasks in punted cgroups it never inited - tolerated + * like the downstream failures of task punting (see scx_punt_task()). + */ +static void scx_cgroup_return_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + bool parent_failed = false; + int ret; + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (tg->scx.sched != sch || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + /* skip the exit if @sch's ops.cgroup_init() failed for the tg */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = parent; + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + /* the first pass must have transferred everything */ + WARN_ON_ONCE(tg->scx.sched == sch); + + /* + * SUB_INIT distinguishes the tgs pass 1 moved. The sched test + * can't: a tg punted to the parent by an earlier failure would + * also match. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + tg->scx.flags &= ~(SCX_TG_SUB_INIT | SCX_TG_INITED); + + /* + * A re-init on $parent failed. The task_groups from here on are + * punted: they stay on the dying $parent with INITED clear and + * move onward when it disables. + */ + if (parent_failed) + continue; + + if (SCX_HAS_OP(parent, cgroup_init)) { + ret = SCX_CALL_OP_RET(parent, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(parent, "ops.cgroup_init() failed (%d) while disabling a sub-scheduler", + ret); + parent_failed = true; + continue; + } + } + tg->scx.flags |= SCX_TG_INITED; + } +} +#else +static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } +static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} +#endif + +void scx_sub_disable(struct scx_sched *sch) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + int ret; + + /* + * Guarantee forward progress and wait for descendants to be disabled. + * To limit disruptions, $parent is not bypassed. Tasks are fully + * prepped and then inserted back into $parent. + */ + scx_bypass(sch, true); + drain_descendants(sch); + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * An enable that failed before scx_link_sched() succeeded never owned a + * cgroup or task and won't be waited on by an ancestor's + * drain_descendants(). Nothing to reparent and walking the tasks can + * misbehave as the task ownership invariant (either owned by self or + * parent) does not hold. ->sibling can't identify this case - an undone + * link leaves it non-empty. + */ + if (!sch->linked) + goto dump; + + set_cgroup_sched(sch_cgroup(sch), parent); + + /* + * Return the subtree's cgroups before re-homing tasks so that any + * ops.init_task() on $parent only sees cgroups it has initialized. + */ + scx_cgroup_return_subtree(sch); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* filter out duplicate visits */ + if (scx_task_on_sched(parent, p)) + continue; + + /* + * By the time control reaches here, all linked descendant + * schedulers should have been disabled. + */ + WARN_ON_ONCE(!scx_task_on_sched(sch, p)); + + /* + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. + */ + get_task_struct(p); + + scx_task_iter_unlock(&sti); + + /* + * $p is READY or ENABLED on @sch. Initialize for $parent, + * disable and exit from @sch, and then switch over to $parent. + * + * If a task fails to initialize for $parent, the only available + * action is disabling $parent too. While this allows disabling + * of a child sched to cause the parent scheduler to fail, the + * failure can only originate from ops.init_task() of the + * parent. A child can't directly affect the parent through its + * own failures. + */ + ret = __scx_init_task(parent, p, NULL, false); + if (ret) { + scx_fail_parent(sch, p, ret); + put_task_struct(p); + break; + } + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on @sch (the + * sched @p was on at that point), not on $parent. + * $parent's just-completed init is owed an exit_task() + * and we issue it here. + */ + scx_sub_init_cancel_task(parent, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + scx_rehome_task(parent, p); + + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + } + scx_task_iter_stop(&sti); + +dump: + scx_disable_dump(sch); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are moved off of @sch but there may still be on-going + * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use + * the expedited version as ancestors may be waiting in bypass mode. + * Also, tell the parent that there is no need to keep running bypass + * DSQs for us. + */ + synchronize_rcu_expedited(); + scx_disable_bypass_dsp(sch); + + scx_unlink_sched(sch); + + mutex_unlock(&scx_enable_mutex); + + /* + * @sch is now unlinked from the parent's children list. Notify and call + * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called + * after unlinking and releasing all locks. See scx_claim_exit(). + */ + wake_up_all(&scx_unlink_waitq); + + if (parent->ops.sub_detach && sch->sub_attached) { + struct scx_sub_detach_args sub_detach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + SCX_CALL_OP(parent, sub_detach, NULL, + &sub_detach_args); + } + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + + /* + * @sch's non-ops programs such as timers and tracers can fire after + * ops.exit(). Now that exit is complete, stop scx_prog_sched() from + * resolving to @sch and drain in-flight resolvers. + */ + WRITE_ONCE(sch->dead, true); + synchronize_rcu(); + + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); + /* not added if enable failed before scx_sched_sysfs_add() */ + if (sch->kobj.state_in_sysfs) + kobject_del(&sch->kobj); +} + +/* verify that a scheduler can be attached to @cgrp and return the parent */ +static struct scx_sched *find_parent_sched(struct cgroup *cgrp) +{ + struct scx_sched *parent = scx_cgroup_sched(cgrp); + struct scx_sched *pos; + + lockdep_assert_held(&scx_sched_lock); + + /* can't attach twice to the same cgroup */ + if (parent->cgrp == cgrp) + return ERR_PTR(-EBUSY); + + /* does $parent allow sub-scheds? */ + if (!parent->ops.sub_attach) + return ERR_PTR(-EOPNOTSUPP); + + /* can't insert between $parent and its exiting children */ + list_for_each_entry(pos, &parent->children, sibling) + if (cgroup_is_descendant(pos->cgrp, cgrp)) + return ERR_PTR(-EBUSY); + + return parent; +} + +static bool assert_task_ready_or_enabled(struct task_struct *p) +{ + u32 state = scx_get_task_state(p); + + switch (state) { + case SCX_TASK_READY: + case SCX_TASK_ENABLED: + return true; + default: + WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", + state, p->comm, p->pid); + return false; + } +} + +void scx_sub_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp; + struct scx_sched *parent, *sch; + struct scx_task_iter sti; + struct task_struct *p; + s32 i, ret; + + mutex_lock(&scx_enable_mutex); + + if (!scx_enabled()) { + ret = -ENODEV; + goto out_unlock; + } + + /* See scx_root_enable_workfn() for the @ops->priv check. */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto out_unlock; + } + + cgrp = cgroup_get_from_id(ops->sub_cgroup_id); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + raw_spin_lock_irq(&scx_sched_lock); + parent = find_parent_sched(cgrp); + if (IS_ERR(parent)) { + raw_spin_unlock_irq(&scx_sched_lock); + ret = PTR_ERR(parent); + goto out_put_cgrp; + } + kobject_get(&parent->kobj); + raw_spin_unlock_irq(&scx_sched_lock); + + /* + * Flip the hot-path gates before ops->priv is published - the sub's + * programs can e.g. kick cpus from that point on. The matching dec is + * at the end of scx_sched_free_rcu_work(). + */ + static_branch_inc(&__scx_has_subs); + + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); + kobject_put(&parent->kobj); + if (IS_ERR(sch)) { + static_branch_dec(&__scx_has_subs); + ret = PTR_ERR(sch); + goto out_unlock; + } + + /* + * Validate before scx_link_sched() publishes @sch, so an invalid sub + * never becomes visible with an unallocated pshard. + */ + ret = scx_validate_ops(sch, ops); + if (ret) + goto err_disable; + + scx_rescue_check_timeout(sch); + + /* + * Allocate pshard[] before scx_link_sched() publishes @sch into the + * parent's RCU children list. A concurrent revoke walking the tree + * would otherwise dereference sch->pshard[si] while it's still NULL. + * Unlike the root path, the cid shard layout is stable at this point. + * + * scx_alloc_pshards() skips allocation when @sch's arena pool isn't + * initialized, so scx_arena_pool_init() must run first. + */ + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_alloc_pshards(sch); + if (ret) + goto err_disable; + + ret = scx_link_sched(sch); + if (ret) + goto err_disable; + + ret = scx_sched_sysfs_add(sch); + if (ret) + goto err_disable; + + if (sch->level >= SCX_SUB_MAX_DEPTH) { + scx_error(sch, "max nesting depth %d violated", + SCX_SUB_MAX_DEPTH); + ret = -EINVAL; + goto err_disable; + } + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = scx_ops_sanitize_err(sch, "init", ret); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + + struct scx_sub_attach_args sub_attach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, + &sub_attach_args); + if (ret) { + ret = scx_ops_sanitize_err(sch, "sub_attach", ret); + scx_error(sch, "parent rejected (%d)", ret); + goto err_disable; + } + sch->sub_attached = true; + + scx_bypass(sch, true); + + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see + * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. + */ + set_cgroup_sched(sch_cgroup(sch), sch); + if (!(cgrp->self.flags & CSS_ONLINE)) { + scx_error(sch, "cgroup is not online"); + ret = -ENODEV; + goto err_unlock_and_disable; + } + + /* + * Take over the subtree's cgroups before any task is claimed, + * mirroring root enable's cgroups-before-tasks order. + */ + ret = scx_cgroup_claim_subtree(sch); + if (ret) + goto err_unlock_and_disable; + + /* + * Initialize tasks for the new child $sch without exiting them for + * $parent so that the tasks can always be reverted back to $parent + * sched on child init failure. + */ + WARN_ON_ONCE(scx_enabling_sub_sched); + scx_enabling_sub_sched = sch; + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* + * Task iteration may visit the same task twice when racing + * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which + * finished __scx_init_task() and skip if set. + * + * A task may exit and get freed between __scx_init_task() + * completion and scx_enable_task(). In such cases, + * scx_disable_and_exit_task() must exit the task for both the + * parent and child scheds. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) + continue; + + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); + + if (!assert_task_ready_or_enabled(p)) { + ret = -EINVAL; + goto abort; + } + + scx_task_iter_unlock(&sti); + + /* + * As $p is still on $parent, it can't be transitioned to INIT. + * Let's worry about task state later. Use __scx_init_task(). + */ + ret = __scx_init_task(sch, p, NULL, false); + if (ret) + goto abort; + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on $parent (the + * sched @p was on at that point), not on @sch. @sch's + * just-completed init is owed an exit_task() and we + * issue it here. + */ + scx_sub_init_cancel_task(sch, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + p->scx.flags |= SCX_TASK_SUB_INIT; + task_rq_unlock(rq, p, &rf); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + /* + * All tasks are prepped. Disable/exit tasks for $parent and enable for + * the new @sch. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * Use clearing of %SCX_TASK_SUB_INIT to detect and skip + * duplicate iterations. + */ + if (!(p->scx.flags & SCX_TASK_SUB_INIT)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p must be either READY or ENABLED. If ENABLED, + * __scx_disabled_and_exit_task() first disables and + * makes it READY. However, after exiting $p, it will + * leave $p as READY. + */ + assert_task_ready_or_enabled(p); + __scx_disable_and_exit_task(parent, p); + + /* + * $p is now only initialized for @sch and READY, which + * is what we want. Assign it to @sch and, if it's on + * the ext class, enable. A non-ext task, possible under + * an %SCX_OPS_SWITCH_PARTIAL root, stays READY and is + * enabled by switching_to_scx() if it switches over. + */ + scx_set_task_sched(p, sch); + if (p->sched_class == &ext_sched_class) + scx_enable_task(sch, p); + + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + + scx_enabling_sub_sched = NULL; + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + /* @sch is enabled; deliver any caps owed since its sub_attach() */ + scx_sub_seed_caps(sch); + + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); + kobject_uevent(&sch->kobj, KOBJ_ADD); + ret = 0; + goto out_unlock; + +out_put_cgrp: + cgroup_put(cgrp); +out_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +abort: + put_task_struct(p); + scx_task_iter_stop(&sti); + + /* + * Undo __scx_init_task() for tasks we marked. scx_enable_task() never + * ran for @sch on them, so calling scx_disable_task() here would invoke + * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched + * must stay set until SUB_INIT is cleared from every marked task - + * scx_disable_and_exit_task() reads it when a task exits concurrently. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (p->scx.flags & SCX_TASK_SUB_INIT) { + scx_sub_init_cancel_task(sch, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + scx_enabling_sub_sched = NULL; +err_unlock_and_disable: + /* we'll soon enter disable path, keep bypass on */ + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + mutex_unlock(&scx_enable_mutex); + /* + * Some enable failures only return an errno (e.g. -ENOMEM from an + * allocation) without calling scx_error(). Record it so + * scx_flush_disable_work() runs the disable and ops.exit() fires. + */ + scx_error(sch, "scx_sub_enable() failed (%d)", ret); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +/** + * scx_cgroup_task_migrating - Prepare a task for a cgroup migration + * @ctx: migration being prepared + * + * A task's sched must match its cgroup's owner, so a migration that crosses a + * sched boundary re-homes the task once committed. Run the fallible part here, + * before the migration commits: initialize the task for the destination sched. + * A rejection fails the cgroup.procs write. + */ +static s32 scx_cgroup_task_migrating(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + int ret; + + /* + * Cleared under scx_cgroup_lock() before root disable starts tearing + * down tasks. As cgroup_mutex is held, a set flag guarantees that the + * teardown loop is not running concurrently. + */ + if (!scx_cgroup_enabled) + return NOTIFY_OK; + + to = scx_cgroup_sched(ctx->dst_dcgrp); + if (scx_task_on_sched(to, p)) + return NOTIFY_OK; + + ret = __scx_init_task(to, p, ctx->dst_dcgrp, false); + if (ret) + return notifier_from_errno(ret); + + return NOTIFY_OK; +} + +/** + * scx_cgroup_task_migrated - Re-home a task that changed cgroups + * @ctx: committed migration + * + * Move the task to its new cgroup's sched, which scx_cgroup_task_migrating() + * already initialized it for. Can't fail. + * + * This is safe against all phases of the destination sched's destruction. A + * disable resets cgroup ownership to the parent and re-homes tasks in one + * scx_cgroup_lock() section. If that section already ran, the destination would + * be the parent. Otherwise, the re-home loop is still ahead and guaranteed to + * visit the task, now in the destination cgroup. + */ +static void scx_cgroup_task_migrated(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + struct rq *rq; + struct rq_flags rf; + + if (!scx_cgroup_enabled) + return; + + to = scx_cgroup_sched(ctx->dst_dcgrp); + if (scx_task_on_sched(to, p)) + return; + + rq = task_rq_lock(p, &rf); + scx_rehome_task(to, p); + task_rq_unlock(rq, p, &rf); +} + +/** + * scx_cgroup_task_migrate_canceled - Undo migration preparation + * @ctx: canceled migration + * + * The migration failed after scx_cgroup_task_migrating() initialized the task + * for the destination sched. The task stays on its current sched in the source + * cgroup. Undo the destination's init. + */ +static void scx_cgroup_task_migrate_canceled(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + struct rq *rq; + struct rq_flags rf; + + if (!scx_cgroup_enabled) + return; + + to = scx_cgroup_sched(ctx->dst_dcgrp); + if (scx_task_on_sched(to, p)) + return; + + rq = task_rq_lock(p, &rf); + scx_sub_init_cancel_task(to, p); + task_rq_unlock(rq, p, &rf); +} + +static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup *cgrp = data; + struct cgroup *parent = cgroup_parent(cgrp); + struct scx_sched *sch; + + if (!cgroup_on_dfl(cgrp)) + return NOTIFY_OK; + + switch (action) { + case CGROUP_LIFETIME_ONLINE: + /* inherit ->scx_sched from $parent */ + if (parent) + rcu_assign_pointer(cgrp->scx_sched, scx_cgroup_sched(parent)); + break; + case CGROUP_LIFETIME_OFFLINE: + /* if there is a sched attached, shoot it down */ + sch = scx_cgroup_sched(cgrp); + if (sch && sch->cgrp == cgrp) + scx_exit(sch, SCX_EXIT_UNREG_KERN, + SCX_ECODE_RSN_CGROUP_OFFLINE, + "cgroup %llu going offline", cgroup_id(cgrp)); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_lifetime_nb = { + .notifier_call = scx_cgroup_lifetime_notify, +}; + +static s32 scx_cgroup_task_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup_task_migrate_ctx *ctx = data; + + switch (action) { + case CGROUP_TASK_MIGRATING: + return scx_cgroup_task_migrating(ctx); + case CGROUP_TASK_MIGRATED: + scx_cgroup_task_migrated(ctx); + break; + case CGROUP_TASK_MIGRATE_CANCELED: + scx_cgroup_task_migrate_canceled(ctx); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_task_nb = { + .notifier_call = scx_cgroup_task_notify, +}; + +static s32 __init scx_cgroup_notifier_init(void) +{ + s32 ret; + + ret = blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); + if (ret) + return ret; + + return blocking_notifier_chain_register(&cgroup_task_notifier, + &scx_cgroup_task_nb); +} +core_initcall(scx_cgroup_notifier_init); + +static void scx_pstack_recursion(struct bpf_prog *prog, const char *op) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(prog->aux); + if (unlikely(!sch)) + return; + + scx_error(sch, "%s recursion detected", op); +} + +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + scx_pstack_recursion(prog, "dispatch"); +} + +void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog) +{ + scx_pstack_recursion(prog, "sub_caps_updated"); +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler + * @cgroup_id: cgroup ID of the child scheduler to dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Allows a parent scheduler to trigger dispatching on one of its direct + * child schedulers. The child scheduler runs its dispatch operation to + * move tasks from dispatch queues to the local runqueue. + * + * Returns: true on success, false if cgroup_id is invalid, not a direct + * child, or caller lacks dispatch permission. + */ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ + struct rq *rq = scx_locked_rq(); + struct scx_sched *parent, *child; + + guard(rcu)(); + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return false; + + child = scx_find_sub_sched(cgroup_id); + + if (unlikely(!child)) + return false; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", + cgroup_id); + return false; + } + + /* + * Skip a child that does not effectively hold the base cap on this cpu: + * its inserts would only be rejected. ecaps are synced at the top of + * dispatch_one() before dispatch, so this reflects the in-effect state. + */ + if (scx_missing_caps(child, cpu_of(rq), SCX_CAP_BASE)) + return false; + + return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) != + SCX_DSP_NONE; +} + +/* Validate common inputs. On success, *parent_out and *child_out are set. */ +static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *aux, + struct scx_sched **parent_out, struct scx_sched **child_out) +{ + struct scx_sched *parent, *child; + + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + child = scx_find_sub_sched(cgroup_id); + if (unlikely(!child)) + return -ENODEV; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "%s: sub-%llu is not a direct child", + parent->cgrp_path, cgroup_id); + return -EINVAL; + } + + if (unlikely(caps & ~__SCX_CAP_ALL)) { + scx_error(parent, "invalid caps 0x%llx", caps); + return -EINVAL; + } + + *parent_out = parent; + *child_out = child; + return 0; +} + +/** + * scx_bpf_sub_grant - Grant @caps on a cmask's cids to a direct child + * @cgroup_id: cgroup id of the direct child sub-sched + * @caps: bitmask of SCX_CAP_* to grant + * @cmask__arena: cid cmask to grant @caps on + * @denied_out__arena__nullable: optional cmask accumulating refused cids + * @aux: implicit BPF argument + * + * A cid in @cmask__arena is granted to the child only if the parent holds every + * requested cap on it. Refused cids are OR'd into the denied mask when + * provided. Refusals outside the denied mask's range are not recorded. + * + * All-or-nothing keeps the caller-visible result binary per cid, so the denied + * mask is one mask to interpret rather than a per-cap matrix. + * + * Return 0 on full success, -EPERM if any cid was refused, or a negative + * errno on other failures. + */ +__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__arena, + struct scx_cmask *denied_out__arena__nullable, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref, denied_ref; + struct scx_sched *parent, *child; + bool any_denied = false; + LIST_HEAD(to_deliver); + s32 si, ret; + + guard(irqsave)(); + + ret = sub_cap_preamble(cgroup_id, caps, aux, &parent, &child); + if (ret) + return ret; + + ret = scx_cmask_ref_init(parent, cmask__arena, &ref); + if (ret) { + scx_error(parent, "invalid cmask (%d)", ret); + return ret; + } + + if (denied_out__arena__nullable) { + ret = scx_cmask_ref_init(parent, denied_out__arena__nullable, &denied_ref); + if (ret) { + scx_error(parent, "invalid denied_out (%d)", ret); + return ret; + } + } + + /* apply the grant one shard at a time */ + for (si = ref.shard_first; si < ref.shard_end; si++) { + SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS); + struct scx_pshard *pps = parent->pshard[si]; + struct scx_pshard *cps = child->pshard[si]; + u64 granted_caps = 0; + u32 cap_bit; + + scx_cmask_ref_shard(&ref, si, slice); + if (scx_cmask_empty(slice)) + continue; + + SCX_CMASK_DEFINE_SHARD(granted_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids); + + scx_cmask_copy(granted_cids, slice); + + scoped_guard (raw_spinlock, &pps->lock) { + guard(raw_spinlock_nested)(&cps->lock); + + /* + * Narrow granted_cids to cids the parent holds every + * requested cap on. All-or-nothing per cid. + */ + scx_for_each_cap_bit(cap_bit, caps) + scx_cmask_and(granted_cids, &pps->caps[cap_bit].cmask); + + /* + * For each requested cap, fold the newly-set cids into + * the child and accumulate the delta. + */ + scx_for_each_cap_bit(cap_bit, caps) { + struct scx_cmask *ccm = &cps->caps[cap_bit].cmask; + + scx_cmask_copy(delta, granted_cids); + scx_cmask_andnot(delta, ccm); + if (scx_cmask_empty(delta)) + continue; + + scx_cmask_or(ccm, delta); + scx_cmask_or(changed_cids, delta); + granted_caps |= BIT_U64(cap_bit); + } + + if (granted_caps) { + s32 cid; + + caps_updated_record(cps, changed_cids, granted_caps, + &to_deliver); + /* + * The sync arms an update_idle() re-notify if + * the cid gains baseline access, so the holder + * learns of an already-idle cid. + */ + scx_cmask_for_each_cid(cid, changed_cids) + queue_sync_ecaps(child, cid); + } + } + + /* record cids that didn't make it into the denied mask */ + if (!scx_cmask_subset(slice, granted_cids)) { + any_denied = true; + if (denied_out__arena__nullable) { + SCX_CMASK_DEFINE_SHARD(denied, slice->base, slice->nr_cids); + + scx_cmask_copy(denied, slice); + scx_cmask_andnot(denied, granted_cids); + scx_cmask_ref_or(&denied_ref, denied); + } + } + } + + caps_updated_deliver(&to_deliver); + + return any_denied ? -EPERM : 0; +} + +/** + * scx_bpf_sub_revoke - Revoke @caps on a cmask's cids from a direct child + * @cgroup_id: cgroup id of the direct child sub-sched + * @caps: bitmask of SCX_CAP_* to revoke + * @cmask__arena: cid cmask to revoke @caps on + * @aux: implicit BPF argument + * + * Clear @caps bits on @cmask__arena from the child named by @cgroup_id and all + * its descendants. The origin parent's pshard lock is held across the subtree + * walk so a concurrent grant from the origin parent observes the revoked state. + */ +__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__arena, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref; + struct scx_sched *parent, *child, *pos; + LIST_HEAD(to_deliver); + s32 si, ret; + + guard(irqsave)(); + + if (sub_cap_preamble(cgroup_id, caps, aux, &parent, &child)) + return; + + ret = scx_cmask_ref_init(parent, cmask__arena, &ref); + if (ret) { + scx_error(parent, "invalid cmask (%d)", ret); + return; + } + + /* per-shard, walk child's subtree and clear @caps */ + for (si = ref.shard_first; si < ref.shard_end; si++) { + SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS); + + scx_cmask_ref_shard(&ref, si, slice); + if (scx_cmask_empty(slice)) + continue; + + /* + * Pre-order with subtree skip: a descendant that cleared + * nothing means no descendant of it can hold @caps on these + * cids either. + */ + guard(raw_spinlock)(&parent->pshard[si]->lock); + pos = scx_next_descendant_pre(NULL, child); + while (pos) { + struct scx_pshard *ps = pos->pshard[si]; + SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids); + u64 revoked_caps = 0; + u32 cap_bit; + + scoped_guard (raw_spinlock_nested, &ps->lock) { + /* + * For each cap, clear lost cids and accumulate + * the per-cap diff for notification. + */ + scx_for_each_cap_bit(cap_bit, caps) { + struct scx_cmask *cm = &ps->caps[cap_bit].cmask; + + scx_cmask_copy(delta, cm); + scx_cmask_and(delta, slice); + if (scx_cmask_empty(delta)) + continue; + + scx_cmask_andnot(cm, delta); + scx_cmask_or(changed_cids, delta); + revoked_caps |= BIT_U64(cap_bit); + } + + if (revoked_caps) { + s32 cid; + + caps_updated_record(ps, changed_cids, revoked_caps, + &to_deliver); + scx_cmask_for_each_cid(cid, changed_cids) + queue_sync_ecaps(pos, cid); + } + } + + if (revoked_caps) + pos = scx_next_descendant_pre(pos, child); + else + pos = scx_skip_subtree_pre(pos, child); + } + } + + caps_updated_deliver(&to_deliver); +} + +/** + * scx_bpf_sub_caps - Read self's or a direct child's cap cmasks + * @cgroup_id: 0 for self, or a direct child's cgroup id + * @caps: one or more SCX_CAP_* bits + * @out__arena: cmask to receive the union of @caps within its range + * @aux: implicit BPF argument + * + * Read the cap cmasks granted on each cid for self (@cgroup_id 0) or a direct + * child - the literal granted set. A sched can read only itself or a direct + * child. + * + * Return 0, -ENODEV if @cgroup_id names no direct child, or -EINVAL on bad + * inputs. + */ +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__arena, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref; + struct scx_sched *sch, *target; + struct scx_pshard **pshard; + s32 si, ret; + + guard(irqsave)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(sch, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + if (unlikely(caps & ~__SCX_CAP_ALL)) { + scx_error(sch, "invalid caps 0x%llx", caps); + return -EINVAL; + } + + /* @cgroup_id 0 reads self, otherwise a direct child */ + if (cgroup_id) { + target = scx_find_sub_sched(cgroup_id); + if (unlikely(!target)) + return -ENODEV; + if (unlikely(scx_parent(target) != sch)) { + scx_error(sch, "%s: sub-%llu is not a direct child", + sch->cgrp_path, cgroup_id); + return -EINVAL; + } + } else { + target = sch; + } + + /* + * The target's caps storage may not be set up yet (e.g. a self-read + * during ops.init_cids()). Pairs with the publish in + * scx_alloc_pshards(): a non-NULL pshard has every element set and the + * acquire also orders the cid table reads below against it. + */ + pshard = smp_load_acquire(&target->pshard); + if (unlikely(!pshard)) { + scx_error(sch, "scx_bpf_sub_caps() called before caps storage is initialized"); + return -ENODEV; + } + + ret = scx_cmask_ref_init(sch, out__arena, &ref); + if (ret) { + scx_error(sch, "invalid out (%d)", ret); + return ret; + } + + for (si = ref.shard_first; si < ref.shard_end; si++) { + const struct scx_cid_shard *shard = + &rcu_dereference_all(scx_cid_shard_ranges)[si]; + SCX_CMASK_DEFINE_SHARD(local_out, shard->base_cid, shard->nr_cids); + u32 cap_bit; + + scx_for_each_cap_bit(cap_bit, caps) + scx_cmask_or(local_out, &pshard[si]->caps[cap_bit].cmask); + scx_cmask_ref_copy(&ref, local_out); + } + return 0; +} + +/** + * scx_bpf_sub_kill_bstr - Kill a direct child sub-scheduler + * @cgroup_id: cgroup id of the direct child to kill + * @fmt: reason message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Evict a direct child sub-scheduler, disabling it with the supplied reason. + * The child and its subtree are torn down asynchronously through the usual + * disable path. + * + * Unlike scx_bpf_exit(), no exit code is taken: the child is a separate + * scheduler with its own exit-code semantics, so a code chosen by the parent + * would have no defined meaning. The reason string carries the intent. + * + * Return 0 on success or -ENODEV if @cgroup_id names no sub-scheduler, which + * can race with the child detaching on its own and so is not a scheduler error. + * Naming a sched that exists but is not a direct child aborts the parent. + */ +__printf(2, 0) +__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *parent, *child; + + guard(rcu)(); + + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + child = scx_find_sub_sched(cgroup_id); + if (unlikely(!child)) + return -ENODEV; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "%s: sub-%llu is not a direct child", + parent->cgrp_path, cgroup_id); + return -EINVAL; + } + + scx_exit_bstr(child, SCX_EXIT_PARENT_KILL, 0, parent, fmt, data, data__sz); + return 0; +} + +__bpf_kfunc_end_defs(); + +#else /* !CONFIG_EXT_SUB_SCHED */ + +__bpf_kfunc_start_defs(); + +__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__arena, + struct scx_cmask *denied_out__arena__nullable, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__arena, + const struct bpf_prog_aux *aux) +{ +} + +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__arena, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc_end_defs(); + +#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h new file mode 100644 index 000000000000..f7bcdfda8dd8 --- /dev/null +++ b/kernel/sched/ext/sub.h @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_SUB_H +#define _KERNEL_SCHED_EXT_SUB_H + +#include "internal.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); +struct cgroup *sch_cgroup(struct scx_sched *sch); +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog); +void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog); +void drain_descendants(struct scx_sched *sch); +void scx_sub_disable(struct scx_sched *sch); +void scx_sub_enable_workfn(struct kthread_work *work); +bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); +void scx_free_pshards(struct scx_sched *sch); +s32 scx_alloc_pshards(struct scx_sched *sch); +void scx_init_root_caps(struct scx_sched *sch); +void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); +void scx_online_ecaps(struct rq *rq); +void scx_offline_ecaps(struct rq *rq); +void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); +void scx_discard_stale_ecaps_syncs(void); +struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); +void scx_reenq_reject(struct rq *rq); +void scx_rescue_charge(struct rq *rq, s64 delta_exec); +void scx_rescue_end(struct rq *rq); +bool scx_rescue_keep(struct rq *rq, struct task_struct *p); +void scx_rescue_flush(struct rq *rq); +void scx_rescue_dump(struct seq_buf *s, struct rq *rq); +void scx_rescue_set_knobs(struct scx_sched *sch); +void scx_rescue_init(struct rq *rq); + +/* + * cgrp->scx_sched is written by root/sub enable/disable under all of + * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the + * parent's sched under just cgroup_mutex but is not yet reachable by the other + * two lock holders. Any one of the three locks stabilizes the association. + */ +static inline struct scx_sched *scx_cgroup_sched(struct cgroup *cgrp) +{ + return rcu_dereference_check(cgrp->scx_sched, + lockdep_is_held(&cgroup_mutex) || + percpu_rwsem_is_held(&scx_fork_rwsem) || + lockdep_is_held(&scx_enable_mutex)); +} + +static inline const char *sch_cgrp_path(struct scx_sched *sch) +{ + return sch->cgrp_path; +} + +/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */ +static inline void scx_dec_has_subs(struct scx_sched *sch) +{ + if (sch->level) + static_branch_dec(&__scx_has_subs); +} + +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } +static inline void scx_free_pshards(struct scx_sched *sch) {} +static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } +static inline void scx_init_root_caps(struct scx_sched *sch) {} +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} +static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_online_ecaps(struct rq *rq) {} +static inline void scx_offline_ecaps(struct rq *rq) {} +static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} +static inline void scx_discard_stale_ecaps_syncs(void) {} +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } +static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_reenq_reject(struct rq *rq) {} +static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} +static inline void scx_rescue_end(struct rq *rq) {} +static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_rescue_flush(struct rq *rq) {} +static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} +static inline void scx_rescue_init(struct rq *rq) {} +static inline void scx_dec_has_subs(struct scx_sched *sch) {} + +#endif /* CONFIG_EXT_SUB_SCHED */ + +/** + * scx_for_each_descendant_pre - pre-order walk of a sched's descendants + * @pos: iteration cursor + * @root: sched to walk the descendants of + * + * Walk @root's descendants. @root is included in the iteration and the first + * node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or + * RCU read lock. + */ +#define scx_for_each_descendant_pre(pos, root) \ + for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ + (pos) = scx_next_descendant_pre((pos), (root))) + +#ifdef CONFIG_EXT_SUB_SCHED + +/** + * scx_missing_caps - The caps in @needed that @sch lacks on @cpu + * @sch: sched to test + * @cpu: cpu to test on + * @needed: bitmask of SCX_CAP_* values + * + * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all. + */ +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) +{ + u64 ecaps; + + /* no sub-scheds, no missing caps */ + if (!scx_has_subs()) + return 0; + + /* root holds every cap on every cpu */ + if (!sch->level) + return 0; + + ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps); + + return needed & ~ecaps; +} + +/* + * Cap semantics: which caps an action requires, and which caps a cap implies. + * Keep all such mappings collected here. + */ + +/* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */ +static inline u64 scx_caps_for_enq(u64 enq_flags) +{ + /* a restored task must be put into the local DSQ regardless of caps */ + if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) + return 0; + if (enq_flags & SCX_ENQ_IMMED) + return SCX_CAP_ENQ_IMMED; + return SCX_CAP_ENQ; +} + +/* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ +static inline u64 scx_caps_for_task(struct task_struct *p) +{ + if (p->scx.flags & SCX_TASK_IMMED) + return SCX_CAP_ENQ_IMMED; + return SCX_CAP_ENQ; +} + +/* the cap @sch needs to preempt @rq's current task, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) +{ + struct task_struct *curr = rq->curr; + + /* a kernel-forced placement preempts regardless of caps */ + if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) + return 0; + /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ + if (curr->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(curr), sch)) + return 0; + return SCX_CAP_PREEMPT; +} + +/* caps implied by holding @cap */ +static inline u64 scx_caps_implied(u64 cap) +{ + switch (cap) { + case SCX_CAP_PREEMPT: + return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED; + case SCX_CAP_ENQ: + return SCX_CAP_ENQ_IMMED; + } + return 0; +} + +/* may @p keep running on @rq's cpu? requires baseline cpu access */ +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) +{ + if (!scx_has_subs()) + return true; + + /* a migration-disabled task is let in without caps, keep it likewise */ + if (unlikely(is_migration_disabled(p))) + return true; + + return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); +} + +/* the task admitted for rescue on @rq, NULL if none */ +static inline struct task_struct *scx_rescuee(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (!scx_has_subs()) + return NULL; + + return rq->scx.rescue.curr; +} + +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } +static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } + +#endif /* CONFIG_EXT_SUB_SCHED */ + +#endif /* _KERNEL_SCHED_EXT_SUB_H */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index bc74eafd43f1..943d8d429a2c 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -19,6 +19,18 @@ enum scx_consts { SCX_DSP_MAX_LOOPS = 32, SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + /* rescue knob defaults and limits, see scx_rescue_timerfn() */ + SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */ + SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */ + SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */ + SCX_RESCUE_DFL_QUANTUM_US = 5000, + SCX_RESCUE_MIN_QUANTUM_US = 1000, + SCX_RESCUE_MAX_QUANTUM_US = 100000, + SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ + SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */ + SCX_RESCUE_MIN_OVERLOAD_MS = 1000, + SCX_RESCUE_MAX_OVERLOAD_MS = 15000, + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ SCX_TID_CHUNK = 1024, @@ -41,17 +53,22 @@ enum scx_consts { SCX_BYPASS_LB_MIN_DELTA_DIV = 4, SCX_BYPASS_LB_BATCH = 256, - SCX_REENQ_LOCAL_MAX_REPEAT = 256, + SCX_REENQ_MAX_REPEAT = 256, SCX_SUB_MAX_DEPTH = 4, }; /* - * Per-cid topology info. For each topology level (core, LLC, node), records - * the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges - * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set - * to -1. + * Per-cid topology info. For each topology level (core, LLC, node) and shard, + * records the first cid in the unit and its global index. Global indices are + * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over + * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields + * set to -1 but always have valid shard assignments. + * + * Shards are contiguous CID ranges used as scalable locking/work domains for + * sub-scheduler operations. By default each LLC becomes one shard, split into + * smaller shards if the LLC exceeds the target size. No-topo cids are packed + * into their own max-sized shards. * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) @@ -59,6 +76,8 @@ enum scx_consts { * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) * @node_cid: first cid of this cid's NUMA node * @node_idx: global index of that node, in [0, nr_nodes_at_init) + * @shard_cid: first cid of this cid's shard + * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) */ struct scx_cid_topo { s32 core_cid; @@ -67,6 +86,24 @@ struct scx_cid_topo { s32 llc_idx; s32 node_cid; s32 node_idx; + s32 shard_cid; + s32 shard_idx; +}; + +enum scx_cid_consts { + SCX_CID_SHARD_SIZE_DFL = 24, + SCX_CID_SHARD_MAX_CPUS = 512, +}; + +/* + * Per-shard metadata for O(1) shard->cid-range lookup. + * + * @base_cid: first cid of the shard + * @nr_cids: number of cids in the shard + */ +struct scx_cid_shard { + s32 base_cid; + s32 nr_cids; }; /* @@ -91,7 +128,7 @@ struct scx_cmask { u32 base; u32 nr_cids; u32 alloc_words; - u64 bits[] __counted_by(alloc_words); + u64 bits[]; }; /* @@ -147,4 +184,41 @@ struct scx_cmask { #define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) +/* + * scx_cmask_ref: validated reference to a BPF-arena cmask. + * + * scx_cmask_ref_init() snapshots @base/@nr_cids. The snapshot is what + * downstream code uses for sizing - the live header can be mutated concurrently + * by BPF. + * + * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and + * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by + * the snapshot. + * + * Typical input use: + * + * struct scx_cmask_ref ref; + * SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS); + * s32 idx, ret; + * + * ret = scx_cmask_ref_init(sch, src, &ref); + * if (ret < 0) + * return ret; + * + * for (idx = ref.shard_first; idx < ref.shard_end; idx++) { + * scx_cmask_ref_shard(&ref, idx, shard); + * if (!shard->nr_cids) + * continue; + * ... use idx and shard ... + * } + */ +struct scx_cmask_ref { + struct scx_sched *sch; + struct scx_cmask *src; + u32 base; + u32 nr_cids; + s32 shard_first; + s32 shard_end; +}; + #endif /* _KERNEL_SCHED_EXT_TYPES_H */ diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 052435f4d3e3..eb73b65ce6c4 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -503,7 +503,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf) { - scx_update_idle(rq, true, false); + /* + * Notify scx only on an idle-to-idle re-pick (the cpu was already idle). + * A real task->idle transition is delivered by set_next_task_idle(), so + * calling here too would duplicate it. + */ + if (scx_enabled() && is_idle_task(rq->curr)) + scx_update_idle(rq, true, false); return rq->idle; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 26ae13c86b69..e656c7059bf8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -787,39 +787,60 @@ enum scx_rq_flags { */ SCX_RQ_ONLINE = 1 << 0, SCX_RQ_CAN_STOP_TICK = 1 << 1, - SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */ SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ + SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ + SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_IN_WAKEUP = 1 << 16, - SCX_RQ_IN_BALANCE = 1 << 17, + SCX_RQ_IN_DISPATCH = 1 << 17, +}; + +/* per-rq rescue execution state, see scx_rescue_timerfn() */ +struct scx_rq_rescue { + struct scx_dispatch_q dsq; /* stranded tasks awaiting rescue */ + s64 budget; /* execution token bucket, ns */ + u64 clock; /* last budget accrual timestamp */ + struct task_struct *curr; /* task being rescued, one at a time */ + s64 slice; /* curr's admitted slice */ + u64 exec_snap; /* sum_exec_runtime at admission */ + struct timer_list timer; /* paces admission and escalation */ + u64 kill_at; /* last ejection, init before any */ }; struct scx_rq { struct scx_dispatch_q local_dsq; +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ + struct scx_rq_rescue rescue; +#endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; - u64 extra_enq_flags; /* see move_task_to_local_dsq() */ + /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ + u64 remote_activate_enq_flags; + struct scx_sched *remote_activate_sch; u32 nr_running; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ bool in_select_cpu; bool cpu_released; u32 flags; u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */ +#ifdef CONFIG_SCHED_CORE + u32 lock_drop_seq; /* nr dispatch lock releases */ +#endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ - cpumask_var_t cpus_to_kick; - cpumask_var_t cpus_to_kick_if_idle; - cpumask_var_t cpus_to_preempt; - cpumask_var_t cpus_to_wait; +#ifdef CONFIG_EXT_SUB_SCHED + struct llist_head ecaps_to_sync; /* pending ecaps syncs */ + struct task_struct *sub_dispatch_prev; +#endif cpumask_var_t cpus_to_sync; bool kick_sync_pending; unsigned long kick_sync; - struct task_struct *sub_dispatch_prev; + struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ raw_spinlock_t deferred_reenq_lock; - u64 deferred_reenq_locals_seq; struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */ struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; @@ -1361,6 +1382,7 @@ struct rq { unsigned int core_forceidle_seq; unsigned int core_forceidle_occupation; u64 core_forceidle_start; + unsigned int core_pick_in_flight; #endif /* CONFIG_SCHED_CORE */ /* Scratch cpumask to be temporarily used under rq_lock */ diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 6b0b4e41b288..69fb4e97bc77 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -668,6 +668,83 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren return found < start ? found : a_end; } +/* + * Like cmask_next_and_set() but over the intersection of THREE masks. Return + * a->base + a->nr_cids if no cid is set in all three at or after @start. + */ +static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b, + const struct scx_cmask __arena *c, + u32 start) +{ + u32 a_end = a->base + a->nr_cids; + u32 b_end = b->base + b->nr_cids; + u32 c_end = c->base + c->nr_cids; + u32 a_wbase = a->base / 64; + u32 b_wbase = b->base / 64; + u32 c_wbase = c->base / 64; + u32 lo = a->base > b->base ? a->base : b->base; + u32 hi = a_end < b_end ? a_end : b_end; + u32 last_wi, start_wi, start_bit, i; + + lo = lo > c->base ? lo : c->base; + hi = hi < c_end ? hi : c_end; + + if (lo >= hi) + return a_end; + if (start < lo) + start = lo; + if (start >= hi) + return a_end; + + last_wi = (hi - 1) / 64; + start_wi = start / 64; + start_bit = start & 63; + + bpf_for(i, 0, CMASK_MAX_WORDS) { + u32 abs_wi = start_wi + i; + u64 word; + u32 found; + + if (abs_wi > last_wi) + break; + + word = a->bits[abs_wi - a_wbase] & b->bits[abs_wi - b_wbase] & + c->bits[abs_wi - c_wbase]; + if (i == 0) + word &= GENMASK_U64(63, start_bit); + if (!word) + continue; + + found = abs_wi * 64 + ctzll(word); + if (found >= hi) + return a_end; + return found; + } + return a_end; +} + +/* + * Round-robin variant of cmask_next_and2_set(): wrap to @a->base if the + * three-way intersection has no cid in the forward half. Return a->base + + * a->nr_cids if empty. + */ +static __always_inline u32 cmask_next_and2_set_wrap(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b, + const struct scx_cmask __arena *c, + u32 start) +{ + u32 a_end = a->base + a->nr_cids; + u32 found; + + found = cmask_next_and2_set(a, b, c, start); + if (found < a_end || start <= a->base) + return found; + + found = cmask_next_and2_set(a, b, c, a->base); + return found < start ? found : a_end; +} + /** * cmask_from_cpumask - translate a kernel cpumask to a cid-space cmask * @m: cmask to fill. Zeroed first; only bits within [@m->base, @m->base + diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 9591a6e778ce..979d4cabfaf9 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -26,6 +26,7 @@ #include #include "user_exit_info.bpf.h" #include "enum_defs.autogen.h" +#include "bpf_arena_common.bpf.h" #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ @@ -96,7 +97,6 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; bool scx_bpf_task_running(const struct task_struct *p) __ksym; s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; -struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym; struct rq *scx_bpf_locked_rq(void) __ksym; struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; @@ -105,7 +105,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __ s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; -s32 scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; @@ -113,7 +113,13 @@ u32 scx_bpf_nr_cids(void) __ksym __weak; u32 scx_bpf_nr_online_cids(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; -void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; + +/* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ +s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; +void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena) __ksym __weak; +s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask __arena *out__arena) __ksym __weak; +s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; /* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from @@ -160,6 +166,22 @@ void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} ___scx_bpf_bstr_format_checker(fmt, ##args); \ }) +/* + * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic + * arguments instead of an array of u64. It kills the direct child sub-scheduler + * @cgid, passing the formatted reason to its user space, and evaluates to the + * kfunc's return value. On a kernel without sub-scheduler support the kfunc is + * absent and it returns -EOPNOTSUPP. + */ +#define scx_bpf_sub_kill(cgid, fmt, args...) \ +({ \ + scx_bpf_bstr_preamble(fmt, args) \ + ___scx_bpf_bstr_format_checker(fmt, ##args); \ + bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \ + scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \ + sizeof(___param)) : -EOPNOTSUPP; \ +}) + /* * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments * instead of an array of u64. Invoking this macro will cause the scheduler to @@ -983,8 +1005,8 @@ extern struct irqtime___local cpu_irqtime __ksym __weak; static inline struct rq___local *get_current_rq(u32 cpu) { /* - * This is a workaround to get an rq pointer since we decided to - * deprecate scx_bpf_cpu_rq(). + * This is a workaround to get an rq pointer now that + * scx_bpf_cpu_rq() has been removed. * * WARNING: The caller must hold the rq lock for @cpu. This is * guaranteed when called from scheduling callbacks (ops.running, diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 3f74d522f7e7..3ab642f92c8a 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -31,7 +31,7 @@ struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak; * * v7.1: scx_bpf_dsq_move_to_local___v2() to add @enq_flags. */ -bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags) __ksym __weak; +bool scx_bpf_dsq_move_to_local___v2___compat(u64 dsq_id, u64 enq_flags) __ksym __weak; bool scx_bpf_dsq_move_to_local___v1(u64 dsq_id) __ksym __weak; void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; @@ -45,8 +45,8 @@ bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct t bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; #define scx_bpf_dsq_move_to_local(dsq_id, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2) ? \ - scx_bpf_dsq_move_to_local___v2((dsq_id), (enq_flags)) : \ + (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2___compat) ? \ + scx_bpf_dsq_move_to_local___v2___compat((dsq_id), (enq_flags)) : \ (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v1) ? \ scx_bpf_dsq_move_to_local___v1((dsq_id)) : \ scx_bpf_consume___old((dsq_id)))) @@ -122,15 +122,20 @@ static inline bool scx_bpf_sub_dispatch(u64 cgroup_id) } /* - * v7.2: scx_bpf_cid_override() for explicit cpu->cid mapping. Ignore if + * v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if * missing. */ -void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) __ksym __weak; +void scx_bpf_cid_override___compat(const s32 __arena *cpu_to_cid__arena, + u32 cpu_to_cid_cnt, + const s32 __arena *shard_start__arena, + u32 shard_start_cnt) __ksym __weak; -static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) +static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_to_cid_cnt, + const s32 __arena *shard_start, u32 shard_start_cnt) { if (bpf_ksym_exists(scx_bpf_cid_override___compat)) - return scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz); + scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid_cnt, + shard_start, shard_start_cnt); } /** @@ -233,23 +238,6 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) -/* - * v6.18: Add a helper to retrieve the current task running on a CPU. - * - * Keep this helper available until v6.20 for compatibility. - */ -static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) -{ - struct rq *rq; - - if (bpf_ksym_exists(scx_bpf_cpu_curr)) - return scx_bpf_cpu_curr(cpu); - - rq = scx_bpf_cpu_rq(cpu); - - return rq ? rq->curr : NULL; -} - /* * v6.19: To work around BPF maximum parameter limit, the following kfuncs are * replaced with variants that pack scalar arguments in a struct. Wrappers are @@ -414,10 +402,10 @@ static inline void scx_bpf_reenqueue_local(void) } /* - * v6.20: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This + * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This * will eventually deprecate scx_bpf_reenqueue_local(). */ -void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux__prog) __ksym __weak; +void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; static inline bool __COMPAT_has_generic_reenq(void) { @@ -427,7 +415,7 @@ static inline bool __COMPAT_has_generic_reenq(void) static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) { if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat)) - scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags, NULL); + scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags); else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0) scx_bpf_reenqueue_local(); else diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 602f07061ee3..d2e4384df5af 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -28,7 +28,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v const struct btf_type *t; const char *n; s32 tid; - int i; + __u32 i; __COMPAT_load_vmlinux_btf(); @@ -42,7 +42,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v if (btf_is_enum(t)) { struct btf_enum *e = btf_enum(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { @@ -53,7 +53,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v } else if (btf_is_enum64(t)) { struct btf_enum64 *e = btf_enum64(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { @@ -85,7 +85,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field const struct btf_member *m; const char *n; s32 tid; - int i; + __u32 i; __COMPAT_load_vmlinux_btf(); tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT); @@ -97,7 +97,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field m = btf_members(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, field)) @@ -154,7 +154,7 @@ static inline long scx_hotplug_seq(void) * struct sched_ext_ops can change over time. Two complementary mechanisms * keep BPF schedulers built against newer headers running on older kernels: * - * 1. Load-time fix-up (this macro). For each optional ops callback or field + * 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field * added to struct sched_ext_ops, an explicit stanza below probes the * running kernel's BTF via __COMPAT_struct_has_field() and, if the field * is missing, clears it in the in-memory struct_ops (with a warning to @@ -175,17 +175,25 @@ static inline long scx_hotplug_seq(void) * - v6.17: ops.cgroup_set_bandwidth() * - v6.19: ops.cgroup_set_idle() * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id + * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ +#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ + struct __scx_name *__oskel; \ + \ + SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ + __ops_struct ".dump() missing, kernel too old?"); \ + \ + __oskel = __scx_name##__open(); \ + SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ + __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ + SCX_ENUM_INIT(__oskel); \ + __oskel; \ +}) + #define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ struct __scx_name *__skel; \ \ - SCX_BUG_ON(!__COMPAT_struct_has_field("sched_ext_ops", "dump"), \ - "sched_ext_ops.dump() missing, kernel too old?"); \ - \ - __skel = __scx_name##__open(); \ - SCX_BUG_ON(!__skel, "Could not open " #__scx_name); \ - __skel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ - SCX_ENUM_INIT(__skel); \ + __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ @@ -211,9 +219,26 @@ static inline long scx_hotplug_seq(void) fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ } \ + if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ + __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ + } \ + if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ + __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ + } \ __skel; \ }) +/* + * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates + * every op the load-time fix-ups above handle, so none of them apply. + */ +#define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ + __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") + /* * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so * that scx_prog_sched() can determine which scheduler a BPF program belongs diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index da4b459820fd..19aa1de3e700 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -7,9 +7,36 @@ #ifndef __ENUM_DEFS_AUTOGEN_H__ #define __ENUM_DEFS_AUTOGEN_H__ +#define HAVE_SCX_ARENA_MIN_ORDER +#define HAVE_SCX_ARENA_GROW_PAGES +#define HAVE___SCX_CAP_ENQ_IMMED +#define HAVE___SCX_CAP_ENQ +#define HAVE___SCX_CAP_PREEMPT +#define HAVE___SCX_CAP_PERF +#define HAVE___SCX_NR_CAPS +#define HAVE___SCX_CAP_ALL +#define HAVE_SCX_CAP_ENQ_IMMED +#define HAVE_SCX_CAP_ENQ +#define HAVE_SCX_CAP_PREEMPT +#define HAVE_SCX_CAP_PERF +#define HAVE_SCX_CAP_BASE +#define HAVE_SCX_CAPS_REENQ_ON_LOSS +#define HAVE_SCX_CID_SHARD_SIZE_DFL +#define HAVE_SCX_CID_SHARD_MAX_CPUS #define HAVE_SCX_DSP_DFL_MAX_BATCH #define HAVE_SCX_DSP_MAX_LOOPS #define HAVE_SCX_WATCHDOG_MAX_TIMEOUT +#define HAVE_SCX_RESCUE_DFL_BW_PPT +#define HAVE_SCX_RESCUE_MAX_BW_PPT +#define HAVE_SCX_RESCUE_DISABLE +#define HAVE_SCX_RESCUE_DFL_QUANTUM_US +#define HAVE_SCX_RESCUE_MIN_QUANTUM_US +#define HAVE_SCX_RESCUE_MAX_QUANTUM_US +#define HAVE_SCX_RESCUE_MIN_SLICE_US +#define HAVE_SCX_RESCUE_OVERLOAD_MULT +#define HAVE_SCX_RESCUE_MIN_OVERLOAD_MS +#define HAVE_SCX_RESCUE_MAX_OVERLOAD_MS +#define HAVE_SCX_TID_CHUNK #define HAVE_SCX_EXIT_BT_LEN #define HAVE_SCX_EXIT_MSG_LEN #define HAVE_SCX_EXIT_DUMP_DFL_LEN @@ -20,7 +47,7 @@ #define HAVE_SCX_BYPASS_LB_DONOR_PCT #define HAVE_SCX_BYPASS_LB_MIN_DELTA_DIV #define HAVE_SCX_BYPASS_LB_BATCH -#define HAVE_SCX_REENQ_LOCAL_MAX_REPEAT +#define HAVE_SCX_REENQ_MAX_REPEAT #define HAVE_SCX_SUB_MAX_DEPTH #define HAVE_SCX_CPU_PREEMPT_RT #define HAVE_SCX_CPU_PREEMPT_DL @@ -35,6 +62,8 @@ #define HAVE_SCX_DSQ_GLOBAL #define HAVE_SCX_DSQ_LOCAL #define HAVE_SCX_DSQ_BYPASS +#define HAVE_SCX_DSQ_REJECT +#define HAVE_SCX_DSQ_RESCUE #define HAVE_SCX_DSQ_LOCAL_ON #define HAVE_SCX_DSQ_LOCAL_CPU_MASK #define HAVE_SCX_DSQ_ITER_REV @@ -53,6 +82,7 @@ #define HAVE_SCX_ENQ_CPU_SELECTED #define HAVE_SCX_ENQ_PREEMPT #define HAVE_SCX_ENQ_IMMED +#define HAVE_SCX_ENQ_RESCUE #define HAVE_SCX_ENQ_REENQ #define HAVE_SCX_ENQ_LAST #define HAVE___SCX_ENQ_INTERNAL_MASK @@ -60,6 +90,9 @@ #define HAVE_SCX_ENQ_DSQ_PRIQ #define HAVE_SCX_ENQ_NESTED #define HAVE_SCX_ENQ_GDSQ_FALLBACK +#define HAVE_SCX_ENQ_IGNORE_CAPS +#define HAVE_SCX_ENQ_APPLY_SLICE +#define HAVE_SCX_ENQ_SLICE_DFL #define HAVE_SCX_TASK_DSQ_ON_PRIQ #define HAVE_SCX_TASK_QUEUED #define HAVE_SCX_TASK_IN_CUSTODY @@ -67,13 +100,16 @@ #define HAVE_SCX_TASK_DEQD_FOR_SLEEP #define HAVE_SCX_TASK_SUB_INIT #define HAVE_SCX_TASK_IMMED +#define HAVE_SCX_TASK_PROTECTED #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK #define HAVE_SCX_TASK_NONE +#define HAVE_SCX_TASK_INIT_BEGIN #define HAVE_SCX_TASK_INIT #define HAVE_SCX_TASK_READY #define HAVE_SCX_TASK_ENABLED +#define HAVE_SCX_TASK_DEAD #define HAVE_SCX_TASK_REENQ_REASON_SHIFT #define HAVE_SCX_TASK_REENQ_REASON_BITS #define HAVE_SCX_TASK_REENQ_REASON_MASK @@ -81,6 +117,7 @@ #define HAVE_SCX_TASK_REENQ_KFUNC #define HAVE_SCX_TASK_REENQ_IMMED #define HAVE_SCX_TASK_REENQ_PREEMPTED +#define HAVE_SCX_TASK_REENQ_CAP #define HAVE_SCX_TASK_CURSOR #define HAVE_SCX_ECODE_RSN_HOTPLUG #define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE @@ -93,17 +130,18 @@ #define HAVE_SCX_EXIT_UNREG_KERN #define HAVE_SCX_EXIT_SYSRQ #define HAVE_SCX_EXIT_PARENT +#define HAVE_SCX_EXIT_PARENT_KILL #define HAVE_SCX_EXIT_ERROR #define HAVE_SCX_EXIT_ERROR_BPF #define HAVE_SCX_EXIT_ERROR_STALL -#define HAVE_SCX_KF_UNLOCKED -#define HAVE_SCX_KF_CPU_RELEASE -#define HAVE_SCX_KF_DISPATCH -#define HAVE_SCX_KF_ENQUEUE -#define HAVE_SCX_KF_SELECT_CPU -#define HAVE_SCX_KF_REST -#define HAVE___SCX_KF_RQ_LOCKED -#define HAVE___SCX_KF_TERMINAL +#define HAVE_SCX_EXIT_ERROR_REENQ +#define HAVE_SCX_EXIT_ERROR_RESCUE +#define HAVE_SCX_KF_ALLOW_UNLOCKED +#define HAVE_SCX_KF_ALLOW_INIT_CIDS +#define HAVE_SCX_KF_ALLOW_CPU_RELEASE +#define HAVE_SCX_KF_ALLOW_DISPATCH +#define HAVE_SCX_KF_ALLOW_ENQUEUE +#define HAVE_SCX_KF_ALLOW_SELECT_CPU #define HAVE_SCX_KICK_IDLE #define HAVE_SCX_KICK_PREEMPT #define HAVE_SCX_KICK_WAIT @@ -121,6 +159,7 @@ #define HAVE_SCX_OPS_ALLOW_QUEUED_WAKEUP #define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE #define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED +#define HAVE_SCX_OPS_TID_TO_TASK #define HAVE_SCX_OPS_ALL_FLAGS #define HAVE___SCX_OPS_INTERNAL_MASK #define HAVE_SCX_OPS_HAS_CPU_PREEMPT @@ -136,6 +175,7 @@ #define HAVE_SCX_SLICE_BYPASS #define HAVE_SCX_SLICE_INF #define HAVE_SCX_REENQ_ANY +#define HAVE_SCX_REENQ_CAP_REVOKE #define HAVE___SCX_REENQ_FILTER_MASK #define HAVE___SCX_REENQ_USER_MASK #define HAVE_SCX_REENQ_TSR_RQ_OPEN @@ -143,14 +183,23 @@ #define HAVE___SCX_REENQ_TSR_MASK #define HAVE_SCX_RQ_ONLINE #define HAVE_SCX_RQ_CAN_STOP_TICK -#define HAVE_SCX_RQ_BAL_KEEP #define HAVE_SCX_RQ_CLK_VALID #define HAVE_SCX_RQ_BAL_CB_PENDING +#define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY +#define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY #define HAVE_SCX_RQ_IN_WAKEUP #define HAVE_SCX_RQ_IN_BALANCE +#define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING +#define HAVE_SCX_SLICE_OOB_DUR_BITS +#define HAVE_SCX_SLICE_OOB_ID_BITS +#define HAVE_SCX_SLICE_OOB_DUR_MASK +#define HAVE_SCX_SLICE_OOB_ID_SHIFT +#define HAVE_SCX_SLICE_OOB_ID_MASK +#define HAVE_SCX_SLICE_OOB_PENDING #define HAVE_SCX_TG_ONLINE #define HAVE_SCX_TG_INITED +#define HAVE_SCX_TG_SUB_INIT #define HAVE_SCX_WAKE_FORK #define HAVE_SCX_WAKE_TTWU #define HAVE_SCX_WAKE_SYNC diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index dafccbb6b69d..7268131010de 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -22,9 +22,6 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING -const volatile u64 __SCX_RQ_BAL_KEEP __weak; -#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP - const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING @@ -37,6 +34,9 @@ const volatile u64 __SCX_RQ_IN_WAKEUP __weak; const volatile u64 __SCX_RQ_IN_BALANCE __weak; #define SCX_RQ_IN_BALANCE __SCX_RQ_IN_BALANCE +const volatile u64 __SCX_RQ_IN_DISPATCH __weak; +#define SCX_RQ_IN_DISPATCH __SCX_RQ_IN_DISPATCH + const volatile u64 __SCX_DSQ_FLAG_BUILTIN __weak; #define SCX_DSQ_FLAG_BUILTIN __SCX_DSQ_FLAG_BUILTIN @@ -124,6 +124,9 @@ const volatile u64 __SCX_ENQ_PREEMPT __weak; const volatile u64 __SCX_ENQ_IMMED __weak; #define SCX_ENQ_IMMED __SCX_ENQ_IMMED +const volatile u64 __SCX_ENQ_RESCUE __weak; +#define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE + const volatile u64 __SCX_ENQ_REENQ __weak; #define SCX_ENQ_REENQ __SCX_ENQ_REENQ diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index bbd4901f4fce..e61632654517 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -11,11 +11,11 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ - SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_BALANCE); \ + SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_DISPATCH); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_INVALID); \ @@ -45,6 +45,7 @@ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \ diff --git a/tools/sched_ext/scx_central.c b/tools/sched_ext/scx_central.c index 4a72df39500d..e1acaed4ec31 100644 --- a/tools/sched_ext/scx_central.c +++ b/tools/sched_ext/scx_central.c @@ -120,7 +120,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_central__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_cpu0.c b/tools/sched_ext/scx_cpu0.c index 4966e3d4c724..1c2b507b8b34 100644 --- a/tools/sched_ext/scx_cpu0.c +++ b/tools/sched_ext/scx_cpu0.c @@ -62,7 +62,7 @@ int main(int argc, char **argv) { struct scx_cpu0 *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); @@ -99,7 +99,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_cpu0__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index fec359581826..64cf4dd964d6 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) * and thus can't be updated and repositioned. Instead, we collect the * vtime deltas separately and apply it asynchronously here. */ - delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0); cvtime = cgv_node->cvtime + delta; /* @@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) cgc = find_cgrp_ctx(cgrp); if (cgc) { __sync_fetch_and_add(&cgc->cvtime_delta, - p->se.sum_exec_runtime - taskc->bypassed_at); + (p->se.sum_exec_runtime - taskc->bypassed_at) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); taskc->bypassed_at = 0; } bpf_cgroup_release(cgrp); @@ -604,6 +605,9 @@ void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) pcgc->child_weight_sum += (s64)weight - cgc->weight; cgc->weight = weight; bpf_spin_unlock(&cgv_tree_lock); + + /* expire cached hweights so the new weight propagates */ + __sync_fetch_and_add(&hweight_gen, 1); } static bool try_pick_next_cgroup(u64 *cgidp) @@ -768,10 +772,18 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) * cgroup to execute but the latter needs to be done in a loop * and we can't keep the lock held. Oh well... */ + s64 delta = now - cpuc->cur_at - cgrp_slice_ns; + bpf_spin_lock(&cgv_tree_lock); - __sync_fetch_and_add(&cgc->cvtime_delta, - (cpuc->cur_at + cgrp_slice_ns - now) * - FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); + /* keep the dividends positive, BPF division is unsigned */ + if (delta >= 0) + __sync_fetch_and_add(&cgc->cvtime_delta, + (u64)delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); + else + __sync_fetch_and_sub(&cgc->cvtime_delta, + (u64)-delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); bpf_spin_unlock(&cgv_tree_lock); } else { stat_inc(FCG_STAT_CNS_GONE); diff --git a/tools/sched_ext/scx_flatcg.c b/tools/sched_ext/scx_flatcg.c index de2bef86d64d..a223bff3746a 100644 --- a/tools/sched_ext/scx_flatcg.c +++ b/tools/sched_ext/scx_flatcg.c @@ -105,12 +105,12 @@ static void fcg_read_stats(struct scx_flatcg *skel, __u64 *stats) __u64 *cnts; __u32 idx; + memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); + cnts = calloc(skel->rodata->nr_cpus, sizeof(__u64)); if (!cnts) return; - memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); - for (idx = 0; idx < FCG_NR_STATS; idx++) { int ret, cpu; @@ -233,7 +233,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_flatcg__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_pair.bpf.c b/tools/sched_ext/scx_pair.bpf.c index 267011b57cba..0d61b7b812db 100644 --- a/tools/sched_ext/scx_pair.bpf.c +++ b/tools/sched_ext/scx_pair.bpf.c @@ -93,12 +93,13 @@ * ----------------------- * * SCX is the lowest priority sched_class, and could be preempted by them at - * any time. To address this, the scheduler implements pair_cpu_release() and - * pair_cpu_acquire() callbacks which are invoked by the core scheduler when - * the scheduler loses and gains control of the CPU respectively. + * any time. To address this, the scheduler watches every sched_switch from + * a tracepoint and edge-detects when a CPU leaves and returns to SCX + * control. * - * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and - * then invoke: + * When a higher-priority class takes a CPU away from a running SCX task - + * a sched_switch from an SCX task to a higher-priority task - we mark the + * pair_ctx as having been preempted and then invoke: * * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); * @@ -107,9 +108,19 @@ * sched_class that preempted our scheduler does not schedule a task * concurrently with our pair CPU. * - * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption - * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable - * pair scheduling. + * When the CPU returns to SCX or idle, we unmark the preemption in the + * pair_ctx and send another resched IPI to the pair CPU to re-enable pair + * scheduling. + * + * A switch from idle straight to a higher-priority task is not a release: + * the CPU was not running an SCX task, so there is nothing to drain and no + * reason to make the pair wait. Kicking SCX_KICK_WAIT on every such wakeup + * would stall the pair CPU behind rt bursts it was never coupled to. + * + * Note that sched_setscheduler() on a running task changes its class in + * place without a context switch, so such transitions are only observed at + * the task's next switch. Until then the stale active_mask bit makes the + * pair wait in try_dispatch(), which is bounded by that next switch. * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo @@ -118,6 +129,8 @@ #include #include "scx_pair.h" +#define MAX_RT_PRIO 100 + char _license[] SEC("license") = "GPL"; /* !0 for veristat, set during init */ @@ -308,6 +321,40 @@ static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) return 0; } +/* + * A task is above SCX whenever its effective priority is in the rt/dl + * range. Test p->prio rather than p->policy: rt_mutex_setprio() boosts + * a PI beneficiary into the rt/dl classes with its policy left + * untouched, so a policy test would misclassify boosted tasks in both + * directions. p->prio follows the boost and the deboost. + * + * This still cannot tell fair and SCX tasks apart. It is complete only + * because scx_pair runs in switch-all mode, where no fair class task + * exists; in partial mode fair is also above SCX and can take the CPU. + */ +static bool pair_task_is_highpri(struct task_struct *p) +{ + return p->prio < MAX_RT_PRIO; +} + +static void pair_cpu_acquire_locked(struct pair_ctx *pairc, u32 in_pair_mask, + u32 *kick_flags) +{ + pairc->preempted_mask &= ~in_pair_mask; + /* Kick the pair CPU, unless it was also preempted. */ + *kick_flags = !pairc->preempted_mask ? SCX_KICK_PREEMPT : 0; +} + +static void pair_cpu_release_locked(struct pair_ctx *pairc, u32 in_pair_mask, + u32 *kick_flags) +{ + pairc->preempted_mask |= in_pair_mask; + pairc->active_mask &= ~in_pair_mask; + /* Kick the pair CPU if it's still running. */ + *kick_flags = pairc->active_mask ? SCX_KICK_PREEMPT | SCX_KICK_WAIT : 0; + pairc->draining = true; +} + __attribute__((noinline)) static int try_dispatch(s32 cpu) { @@ -500,61 +547,60 @@ void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) } } -void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) +SEC("tp_btf/sched_switch") +int BPF_PROG(pair_sched_switch, bool preempt, struct task_struct *prev, + struct task_struct *next, unsigned int prev_state) { int ret; + s32 cpu = bpf_get_smp_processor_id(); u32 in_pair_mask; struct pair_ctx *pairc; - bool kick_pair; + u32 kick_flags = 0; + bool preempted; + bool release, acquire; ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); if (ret) - return; + return 0; + + /* + * This runs on every context switch in the system. A CPU's own + * preempted_mask bit is only ever written by this tracepoint + * running on that CPU, so the unlocked read is exact and the + * pair-shared lock is only taken on actual transitions. + */ + preempted = pairc->preempted_mask & in_pair_mask; + if (next->pid && pair_task_is_highpri(next)) { + /* an SCX task lost the CPU to a higher-priority class */ + release = !preempted && prev->pid && !pair_task_is_highpri(prev); + acquire = false; + } else { + /* the CPU is back under SCX control (or idle) */ + release = false; + acquire = preempted; + } + if (!release && !acquire) + return 0; bpf_spin_lock(&pairc->lock); - pairc->preempted_mask &= ~in_pair_mask; - /* Kick the pair CPU, unless it was also preempted. */ - kick_pair = !pairc->preempted_mask; + if (release) { + pair_cpu_release_locked(pairc, in_pair_mask, &kick_flags); + __sync_fetch_and_add(&nr_preemptions, 1); + } else { + pair_cpu_acquire_locked(pairc, in_pair_mask, &kick_flags); + } bpf_spin_unlock(&pairc->lock); - if (kick_pair) { + if (kick_flags) { s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids); if (pair) { __sync_fetch_and_add(&nr_kicks, 1); - scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); + scx_bpf_kick_cpu(*pair, kick_flags); } } -} -void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) -{ - int ret; - u32 in_pair_mask; - struct pair_ctx *pairc; - bool kick_pair; - - ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); - if (ret) - return; - - bpf_spin_lock(&pairc->lock); - pairc->preempted_mask |= in_pair_mask; - pairc->active_mask &= ~in_pair_mask; - /* Kick the pair CPU if it's still running. */ - kick_pair = pairc->active_mask; - pairc->draining = true; - bpf_spin_unlock(&pairc->lock); - - if (kick_pair) { - s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids); - - if (pair) { - __sync_fetch_and_add(&nr_kicks, 1); - scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); - } - } - __sync_fetch_and_add(&nr_preemptions, 1); + return 0; } s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) @@ -602,8 +648,6 @@ void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) SCX_OPS_DEFINE(pair_ops, .enqueue = (void *)pair_enqueue, .dispatch = (void *)pair_dispatch, - .cpu_acquire = (void *)pair_cpu_acquire, - .cpu_release = (void *)pair_cpu_release, .cgroup_init = (void *)pair_cgroup_init, .cgroup_exit = (void *)pair_cgroup_exit, .exit = (void *)pair_exit, diff --git a/tools/sched_ext/scx_pair.c b/tools/sched_ext/scx_pair.c index 41b136d43a55..00f595b58f97 100644 --- a/tools/sched_ext/scx_pair.c +++ b/tools/sched_ext/scx_pair.c @@ -190,7 +190,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_pair__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index fd9a82a67627..5bb8b90a275a 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1,21 +1,39 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * A simple five-level FIFO queue scheduler. + * scx_qmap: a demonstration and testing scheduler for sched_ext features. * - * There are five FIFOs implemented as arena-backed doubly-linked lists - * threaded through per-task context. A task gets assigned to one depending on - * its compound weight. Each CPU round robins through the FIFOs and dispatches - * more from FIFOs with higher indices - 1 from queue0, 2 from queue1, 4 from - * queue2 and so on. - * - * This scheduler demonstrates: + * A simple scheduler that exercises a broad set of sched_ext features. Unlikely + * to be useful for real workloads. It demonstrates: * * - BPF-side queueing using TIDs. * - BPF arena for scheduler state. * - Core-sched support. + * - Hierarchical sub-scheduling: delegating cpus to child cgroup schedulers. * - * This scheduler is primarily for demonstration and testing of sched_ext - * features and unlikely to be useful for actual workloads. + * Base design: Five FIFOs (arena-backed doubly-linked lists through per-task + * context). A task is assigned to a FIFO by its compound weight. Each cpu + * round-robins the FIFOs, dispatching more from higher ones. + * + * Sub-scheduling: Any qmap sched can delegate cpus to its own child cgroup + * schedulers and keep the rest for its tasks. Terminology: + * + * excl - A cpu the delegatee owns wholly (ENQ_IMMED|ENQ|PREEMPT). + * shared - A cpu delegated as ENQ_IMMED only. Time-shared. + * held_excl / held_shared - What this node was handed by its parent. + * held-excl cpus are re-delegatable. A held-shared cpu is a + * time-share that stays self-local. + * self - The excl cpus the node kept for itself, plus all of held_shared. + * owner - Who holds a cid - a child slot, CID_SELF, or CID_NONE. + * + * The scheduler splits its held-excl cpus among self and the children in + * proportion to each node's cpu.weight, handing each the floor of its share as + * excl cpus. The leftover from rounding forms a shared pool the round-robin + * timer hands around. With no excl cpu to delegate, the node evicts its + * children. + * + * This policy is a demonstration only, not a practical one. The split + * considers only direct children and is not work-conserving. It only exists to + * drive sub-sched primitives with as simple logic as possible. * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo @@ -48,28 +66,20 @@ const volatile bool print_msgs; const volatile u64 sub_cgroup_id; const volatile s32 disallow_tgid; const volatile bool suppress_dump; -const volatile bool always_enq_immed; const volatile u32 immed_stress_nth; const volatile u32 max_tasks; +/* sub-sched: period for handing the round-robin cid pool to the next child */ +const volatile u64 round_robin_ns; + /* * Optional cid-override test harness. When cid_override_mode is non-zero, - * qmap_init() calls scx_bpf_cid_override() with the caller-supplied - * cpu_to_cid array to exercise the kfunc's acceptance and error paths. - * - * 0 = disabled - * 1 = valid reverse mapping - * 2 = invalid: duplicate cid assignment - * 3 = invalid: out-of-range cid + * qmap_init_cids() calls scx_bpf_cid_override() with the caller-supplied arrays + * to exercise the kfunc's acceptance and error paths. See enum + * qmap_cid_override for the modes. */ const volatile u32 cid_override_mode; -/* - * Array lives in bss (writable) because scx_bpf_cid_override()'s BPF - * verifier signature treats its len-paired pointer as read/write - rodata - * fails verification with "write into map forbidden". Userspace populates - * it before SCX_OPS_LOAD, same as rodata, and nothing writes it after. - */ -s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; +const volatile u32 cid_override_nr_shards; UEI_DEFINE(uei); @@ -91,12 +101,12 @@ struct { struct qmap_arena __arena_global qa; -/* - * Global idle-cid tracking, maintained via update_idle / cpu_offline and - * scanned by the direct-dispatch path. Allocated in qmap_init() from one - * arena page, sized to the full cid space. - */ -struct scx_cmask __arena *qa_idle_cids; +/* ensure that BPF and userspace are seeing the same size for qmap_cmask */ +_Static_assert(QMAP_CMASK_WORDS == CMASK_NR_WORDS(SCX_QMAP_MAX_CPUS), + "QMAP_CMASK_WORDS must equal CMASK_NR_WORDS(SCX_QMAP_MAX_CPUS)"); +_Static_assert(sizeof(struct qmap_cmask) == + struct_size_t(struct scx_cmask, bits, QMAP_CMASK_WORDS), + "qmap_cmask must be exactly sized to back a full scx_cmask"); /* Per-queue locks. Each in its own .data section as bpf_res_spin_lock requires. */ __hidden struct bpf_res_spin_lock qa_q_lock0 SEC(".data.qa_q_lock0"); @@ -198,7 +208,7 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock) } /* - * Try prev_cid, then scan taskc->cpus_allowed AND qa_idle_cids round-robin + * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin * from prev_cid + 1. Atomic claim retries on race; bounded by * IDLE_PICK_RETRIES to keep the verifier's insn budget in check. */ @@ -211,20 +221,19 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, s32 cid; u32 i; - if (!always_enq_immed && p->nr_cpus_allowed == 1) - return prev_cid; - - if (cmask_test_and_clear(prev_cid, qa_idle_cids)) + if (cmask_test(prev_cid, &qa.self_cids.mask) && + cmask_test_and_clear(prev_cid, &qa.idle_cids.mask)) return prev_cid; cid = prev_cid; bpf_for(i, 0, IDLE_PICK_RETRIES) { - cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, - qa_idle_cids, cid + 1); + cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, + &qa.idle_cids.mask, + &qa.self_cids.mask, cid + 1); barrier_var(cid); if (cid >= nr_cids) return -1; - if (cmask_test_and_clear(cid, qa_idle_cids)) + if (cmask_test_and_clear(cid, &qa.idle_cids.mask)) return cid; } return -1; @@ -348,6 +357,33 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p, } } +/* + * A received time-shared cid is held ENQ_IMMED-only, so inserts must set + * SCX_ENQ_IMMED. + */ +static u64 needs_immed(s32 cid) +{ + return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0; +} + +/* first cid this node does NOT hold for fault injection, -1 if none */ +static s32 first_unavail_cid(void) +{ + s32 nr_cids = qa.nr_cids, c; + + if (nr_cids > SCX_QMAP_MAX_CPUS) { + scx_bpf_error("-ERANGE"); + return -1; + } + + bpf_for(c, 0, nr_cids) { + if (!cmask_test(c, &qa.held_excl.mask) && + !cmask_test(c, &qa.held_shared.mask)) + return c; + } + return -1; +} + static int weight_to_idx(u32 weight) { /* Coarsely map the compound weight to a FIFO. */ @@ -371,9 +407,16 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) s32 cid; if (enq_flags & SCX_ENQ_REENQ) { + u64 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; + __sync_fetch_and_add(&qa.nr_reenqueued, 1); if (scx_bpf_task_cid(p) == 0) __sync_fetch_and_add(&qa.nr_reenqueued_cid0, 1); + /* cap-loss and IMMED-handback bounces, relocated below */ + if (reason == SCX_TASK_REENQ_CAP) + __sync_fetch_and_add(&qa.nr_reenq_cap, 1); + else if (reason == SCX_TASK_REENQ_IMMED) + __sync_fetch_and_add(&qa.nr_reenq_immed, 1); } if (p->flags & PF_KTHREAD) { @@ -396,6 +439,52 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) */ taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++; + /* + * A task of ours that can run on none of our self cids - the parent + * didn't grant them or we delegated them to children - would starve in + * SHARED/FIFO since we only pull from those on self cids. + * + * Force it onto its first allowed cid's local DSQ. If we hold that cid + * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel + * diverts the task to its rescue path. + */ + if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { + s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + + if (c >= 0 && c < scx_bpf_nr_cids()) { + taskc->force_local = false; + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, + enq_flags | needs_immed(c) | SCX_ENQ_RESCUE); + return; + } + } + + /* + * Fault injection: deliberately dispatch one of our own tasks to a cid + * we don't hold. The inserts carry SCX_ENQ_RESCUE and divert to the + * kernel rescue path, a deterministic rescue-traffic generator. Under + * -B 0 the kernel cap check rejects and re-enqueues them instead, so + * nr_inject_attempts tracks nr_reenq_cap 1:1 and proves delivery-time + * enforcement. Throttled. + */ + if (qa.inject_mode == QMAP_INJ_WRONG_CID && p->nr_cpus_allowed > 1 && + !(enq_flags & SCX_ENQ_REENQ)) { + static u32 inj_cnt; + + if (!(++inj_cnt % 64)) { + s32 bad = first_unavail_cid(); + + if (bad >= 0 && cmask_test(bad, &taskc->cpus_allowed)) { + __sync_fetch_and_add(&qa.nr_inject_attempts, 1); + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | bad, slice_ns, + enq_flags | SCX_ENQ_RESCUE); + return; + } + } + } + /* * IMMED stress testing: Every immed_stress_nth'th enqueue, dispatch * directly to prev_cpu's local DSQ even when busy to force dsq->nr > 1 @@ -418,7 +507,8 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) */ if (taskc->force_local) { taskc->force_local = false; - scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice_ns, + enq_flags | needs_immed(scx_bpf_task_cid(p))); return; } @@ -433,7 +523,8 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) if (!__COMPAT_is_enq_cpu_selected(enq_flags) && (cid = pick_direct_dispatch_cid(p, scx_bpf_task_cid(p), taskc)) >= 0) { __sync_fetch_and_add(&qa.nr_ddsp_from_enq, 1); - scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, enq_flags); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | needs_immed(cid)); return; } @@ -447,8 +538,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) s32 cid; scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); - cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, - qa_idle_cids, 0); + cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, + &qa.idle_cids.mask, + &qa.self_cids.mask, 0); if (cid < scx_bpf_nr_cids()) scx_bpf_kick_cid(cid, SCX_KICK_IDLE); return; @@ -490,29 +582,47 @@ static void update_core_sched_head_seq(struct task_struct *p) } /* - * To demonstrate the use of scx_bpf_dsq_move(), implement silly selective - * priority boosting mechanism by scanning SHARED_DSQ looking for highpri tasks, - * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor - * difference only when dsp_batch is larger than 1. + * One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A + * task whose cids were lost while it was queued in the fifos would strand on + * SHARED_DSQ, which is consumed only on self cids it can't run on - move it to + * the kernel rescue path. One whose cids were lost after the highpri cull is + * likewise rescued out of HIGHPRI_DSQ below. * - * scx_bpf_dispatch[_vtime]_from_dsq() are allowed both from ops.dispatch() and + * To demonstrate the use of scx_bpf_dsq_move(), implement silly selective + * priority boosting mechanism by moving highpri tasks to HIGHPRI_DSQ and then + * consuming them first. This makes minor difference only when dsp_batch is + * larger than 1. + * + * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and * non-rq-lock holding BPF programs. As demonstration, this function is called * from qmap_dispatch() and monitor_timerfn(). */ -static bool dispatch_highpri(bool from_timer) +static bool scan_shared_dsq(bool from_timer) { struct task_struct *p; s32 this_cid = scx_bpf_this_cid(); u32 nr_cids = scx_bpf_nr_cids(); - /* scan SHARED_DSQ and move highpri tasks to HIGHPRI_DSQ */ + /* rescue strands and move highpri tasks to HIGHPRI_DSQ */ bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) { static u64 highpri_seq; task_ctx_t *taskc; + s32 c; if (!(taskc = lookup_task_ctx(p))) return false; + /* stranded? rescue - it can't be dispatched here either way */ + if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + if (c >= 0 && c < scx_bpf_nr_cids()) { + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, + needs_immed(c) | SCX_ENQ_RESCUE); + } + continue; + } + if (taskc->highpri) { /* exercise the set_*() and vtime interface too */ scx_bpf_dsq_move_set_slice(BPF_FOR_EACH_ITER, slice_ns * 2); @@ -534,16 +644,28 @@ static bool dispatch_highpri(bool from_timer) if (!(taskc = lookup_task_ctx(p))) return false; - if (cmask_test(this_cid, &taskc->cpus_allowed)) + /* only run highpri tasks on cids this node holds, not delegated ones */ + if (cmask_test(this_cid, &taskc->cpus_allowed) && + cmask_test(this_cid, &qa.self_cids.mask)) cid = this_cid; else - cid = cmask_next_set_wrap(&taskc->cpus_allowed, - this_cid + 1); - if (cid >= nr_cids) + cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, + &qa.self_cids.mask, + this_cid + 1); + if (cid >= nr_cids) { + /* stranded after the cull - rescue it from here */ + s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + + if (c >= 0 && c < nr_cids) { + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, + needs_immed(c) | SCX_ENQ_RESCUE); + } continue; + } if (scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | cid, - SCX_ENQ_PREEMPT)) { + SCX_ENQ_PREEMPT | needs_immed(cid))) { if (cid == this_cid) { dispatched = true; __sync_fetch_and_add(&qa.nr_expedited_local, 1); @@ -569,12 +691,37 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) struct cpu_ctx __arena *cpuc; task_ctx_t *taskc; u32 batch = dsp_batch ?: 1; - s32 i; + s32 owner, i; - if (dispatch_highpri(false)) + if (scan_shared_dsq(false)) return; - if (!qa.nr_highpri_queued && scx_bpf_dsq_move_to_local(SHARED_DSQ, 0)) + /* + * Sub-sched routing: a child-owned cid goes to its owner. Never run + * this node's own tasks on a delegated cid. Read without the guard. + */ + owner = qa.part.cid_owner[cid]; + if (owner == CID_SHARED) { + /* route to the live rr holder (0 = self, runs below) */ + s32 pos = qa.part.rr_pos; + u64 holder_cgid = (pos >= 0 && pos < MAX_PARTS) ? + qa.part.rr_slots[pos] : 0; + + if (holder_cgid) { + scx_bpf_sub_dispatch(holder_cgid); + return; + } + } else if (owner >= 0 && owner < MAX_SUB_SCHEDS) { + u64 cgid = qa.sub_sched_ctxs[owner].cgroup_id; + + if (cgid) { + if (scx_bpf_sub_dispatch(cgid)) + __sync_fetch_and_add(&qa.sub_sched_ctxs[owner].nr_dsps, 1); + return; + } + } + + if (!qa.nr_highpri_queued && scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid))) return; if (dsp_inf_loop_after && qa.nr_dispatched > dsp_inf_loop_after) { @@ -658,13 +805,12 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) */ if (!cmask_test(cid, &taskc->cpus_allowed)) scx_bpf_kick_cid(scx_bpf_task_cid(p), 0); - batch--; cpuc->dsp_cnt--; if (!batch || !scx_bpf_dispatch_nr_slots()) { - if (dispatch_highpri(false)) + if (scan_shared_dsq(false)) return; - scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); + scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)); return; } if (!cpuc->dsp_cnt) @@ -674,11 +820,8 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) cpuc->dsp_cnt = 0; } - for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (qa.sub_sched_cgroup_ids[i] && - scx_bpf_sub_dispatch(qa.sub_sched_cgroup_ids[i])) - return; - } + if (scan_shared_dsq(false)) + return; /* * No other tasks. @prev will keep running. Update its core_sched_seq as @@ -715,16 +858,11 @@ void BPF_STRUCT_OPS(qmap_tick, struct task_struct *p) * The distance from the head of the queue scaled by the weight of the queue. * The lower the number, the older the task and the higher the priority. */ -static s64 task_qdist(struct task_struct *p) +static s64 task_qdist(struct task_struct *p, task_ctx_t *taskc) { int idx = weight_to_idx(p->scx.weight); - task_ctx_t *taskc; s64 qdist; - taskc = lookup_task_ctx(p); - if (!taskc) - return 0; - qdist = taskc->core_sched_seq - qa.core_sched_head_seqs[idx]; /* @@ -749,7 +887,21 @@ static s64 task_qdist(struct task_struct *p) bool BPF_STRUCT_OPS(qmap_core_sched_before, struct task_struct *a, struct task_struct *b) { - return task_qdist(a) > task_qdist(b); + task_ctx_t *taskc_a = lookup_task_ctx(a); + task_ctx_t *taskc_b = lookup_task_ctx(b); + + /* + * A task delegated to a sub-scheduler has no task_ctx here. Order such + * pairs by the kernel's default ordering - a running task after every + * waiting task, then by runnable_at. + */ + if (!taskc_a || !taskc_b) { + if (a->on_cpu != b->on_cpu) + return b->on_cpu; + return time_before(a->scx.runnable_at, b->scx.runnable_at); + } + + return task_qdist(a, taskc_a) < task_qdist(b, taskc_b); } /* @@ -764,6 +916,10 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_task, struct task_struct *p, struct task_ctx_stor_val *v; task_ctx_t *taskc; + if (qa.inject_mode == QMAP_INJ_INIT_FAIL && + !bpf_strncmp(p->comm, 6, "qmfail")) + return -ENOMEM; + if (p->tgid == disallow_tgid) p->scx.disallow = true; @@ -886,36 +1042,85 @@ void BPF_STRUCT_OPS(qmap_dump_task, struct scx_dump_ctx *dctx, struct task_struc taskc->force_local, taskc->core_sched_seq); } -s32 BPF_STRUCT_OPS(qmap_cgroup_init, struct cgroup *cgrp, struct scx_cgroup_init_args *args) +s32 BPF_STRUCT_OPS(qmap_cpuctl_init, struct cgroup *cgrp, struct scx_cgroup_init_args *args) { + QMAP_TOUCH_ARENA(); + if (print_msgs) bpf_printk("CGRP INIT %llu weight=%u period=%lu quota=%ld burst=%lu", cgrp->kn->id, args->weight, args->bw_period_us, args->bw_quota_us, args->bw_burst_us); + + if (qa.inject_mode == QMAP_INJ_CGRP_INIT_FAIL) { + char name[7] = {}; + + bpf_probe_read_kernel_str(name, sizeof(name), cgrp->kn->name); + if (!bpf_strncmp(name, 6, "qmfail")) + return -ENOMEM; + } + return 0; } -void BPF_STRUCT_OPS(qmap_cgroup_set_weight, struct cgroup *cgrp, u32 weight) +static void redistribute(void); + +void BPF_STRUCT_OPS(qmap_cpuctl_set_weight, struct cgroup *cgrp, u32 weight) { + u64 cgid = cgrp->kn->id; + s32 i; + + QMAP_TOUCH_ARENA(); + if (print_msgs) - bpf_printk("CGRP SET %llu weight=%u", cgrp->kn->id, weight); + bpf_printk("CGRP SET %llu weight=%u", cgid, weight); + + /* + * Knobs belong to the parent, so this op carries the child subs' + * attach point weights. Adjust the matching sub's share of the cid + * partition. Other cgroups don't participate in the split. + */ + for (i = 0; i < MAX_SUB_SCHEDS; i++) { + if (qa.sub_sched_ctxs[i].cgroup_id != cgid) + continue; + if (qa.sub_sched_ctxs[i].weight != weight) { + qa.sub_sched_ctxs[i].weight = weight; + redistribute(); + } + break; + } } -void BPF_STRUCT_OPS(qmap_cgroup_set_bandwidth, struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us) +void BPF_STRUCT_OPS(qmap_cpuctl_set_bandwidth, struct cgroup *cgrp, u64 period_us, + u64 quota_us, u64 burst_us) { if (print_msgs) bpf_printk("CGRP SET %llu period=%lu quota=%ld burst=%lu", cgrp->kn->id, period_us, quota_us, burst_us); } +void BPF_STRUCT_OPS(qmap_cpuctl_move, struct task_struct *p, struct cgroup *from, + struct cgroup *to) +{ + if (print_msgs) + bpf_printk("CGRP MOVE %d %llu -> %llu", + p->pid, from->kn->id, to->kn->id); +} + void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) { QMAP_TOUCH_ARENA(); + + /* + * The kernel delivers update_idle() for every cid this node holds + * SCX_CAP_BASE on. Track every cid's idle state regardless of + * delegation: the direct-dispatch pick masks idle_cids with self_cids + * at selection, so a cid already idle when it returns to self needs no + * reseed here. + */ if (idle) - cmask_set(cid, qa_idle_cids); + cmask_set(cid, &qa.idle_cids.mask); else - cmask_clear(cid, qa_idle_cids); + cmask_clear(cid, &qa.idle_cids.mask); } void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, @@ -1011,7 +1216,7 @@ static void dump_shared_dsq(void) static int monitor_timerfn(void *map, int *key, struct bpf_timer *timer) { bpf_rcu_read_lock(); - dispatch_highpri(true); + scan_shared_dsq(true); bpf_rcu_read_unlock(); monitor_cpuperf(); @@ -1067,6 +1272,508 @@ static int lowpri_timerfn(void *map, int *key, struct bpf_timer *timer) return 0; } +struct round_robin_timer { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct round_robin_timer); +} round_robin_timer SEC(".maps"); + +/* + * Partition update synchronization. qa.part can be written from concurrent + * contexts. This single-runner guard admits one writer at a time without + * holding a lock across the grant/revoke kfuncs. part_pending coalesces + * repartition requests that arrive while it is held. + * + * They live in .bss, not the arena: rr_advance() runs from a bpf_timer + * callback, where the verifier rejects atomic ops on arena memory. + */ +static u64 part_busy; +static u64 part_pending; + +static bool part_try_start(void) +{ + /* set busy, report whether it was previously clear (we acquired it) */ + return !__sync_fetch_and_or(&part_busy, 1); +} + +static void part_end(void) +{ + __sync_fetch_and_and(&part_busy, 0); +} + +/* + * compute_partition() scratch. + * + * The excl-held cids are handed out in cid order: position 0..nr_excl-1 over + * the held cids is split into contiguous ranges, one per participant that gets + * at least one excl cid. Range k is owned by cp_range_owner[k] and ends at the + * cumulative position cp_range_end[k]. + */ +static s32 cp_range_owner[MAX_PARTS]; /* exclusive range k: its owner id ... */ +static s32 cp_range_end[MAX_PARTS]; /* ... and the cumulative position it ends at */ + +/* a participant in the partition: self or an attached child */ +struct participant { + s32 slot; /* child slot, or CID_SELF */ + u32 weight; /* cpu.weight */ +}; + +/** + * place_one - assign one excl-held cid to its owner + * @cid: the excl-held cid to place + * @n: its position among the excl-held cids, in [0, nr_excl) + * @total_excl: how many positions are owned exclusively (the rest are shared) + * + * Position @n below @total_excl is owned exclusively. It falls in the range + * whose cumulative end it is under, owned by cp_range_owner[]. A position at or + * above @total_excl is the rounding leftover which joins the shared pool. + * + * A separate __noinline function to help verification. + */ +__noinline int place_one(s32 cid, s32 n, s32 total_excl) +{ + s32 owner = CID_SELF, i, s; + + if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS || n < 0 || n >= SCX_QMAP_MAX_CPUS || + total_excl < 0) { + scx_bpf_error("-ERANGE"); + return 0; + } + + if (n < total_excl) { + for (i = 0; i < MAX_PARTS; i++) { + if (n < cp_range_end[i]) { + owner = cp_range_owner[i]; + break; + } + } + qa.part.cid_owner[cid] = owner; + } else { + s = n - total_excl; + if (s < 0 || s >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return 0; + } + qa.part.shared_cids[s] = cid; + /* time-shared: dispatch resolves the live holder via rr_pos */ + qa.part.cid_owner[cid] = CID_SHARED; + } + return 0; +} + +/** + * compute_partition - build the cid partition from this node's held caps + * + * Decide each cid's owner, the shared pool and the rr rotation. __noinline to + * help verification. See the comment at the top of the file. + */ +__noinline void compute_partition(void) +{ + s32 nr_cids = qa.nr_cids; + s32 nr_excl, total_excl = 0, nr_rr = 0; + s32 sum_w, i, cid, n = 0, share, self_w; + u64 cgid_snap[MAX_SUB_SCHEDS]; + s32 w_snap[MAX_SUB_SCHEDS]; + + if (nr_cids > SCX_QMAP_MAX_CPUS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* find out the cids we hold */ + scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); + cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */ + + qa.part.nr_shared = 0; + qa.part.nr_rr = 0; + qa.part.rr_pos = 0; + + nr_excl = cmask_weight(&qa.held_excl.mask); + qa.part.nr_excl = nr_excl; + + /* no excl cid: held_shared stays self-local, the rest unheld */ + if (!nr_excl) { + bpf_for(cid, 0, nr_cids) { + if (cmask_test(cid, &qa.held_shared.mask)) + qa.part.cid_owner[cid] = CID_SELF; + else + qa.part.cid_owner[cid] = CID_NONE; + } + return; + } + + /* + * Snapshot membership and weights so the sum_w and share loops agree. A + * mid-compute change would otherwise wrap nr_shared negative. The self + * weight is fixed at the default: a cgroup's weight is its parent's + * knob, not the scheduler's own business. + */ + self_w = 100; + bpf_for(i, 0, MAX_SUB_SCHEDS) { + cgid_snap[i] = qa.sub_sched_ctxs[i].cgroup_id; + w_snap[i] = cgid_snap[i] ? (qa.sub_sched_ctxs[i].weight ?: 100) : 0; + } + + /* + * Participants are self plus each child. Give each a fixed range/rr + * slot: self at slot 0, child i at slot i+1. + * + * sum_w totals every participant's weight. + */ + sum_w = self_w; + bpf_for(i, 0, MAX_SUB_SCHEDS) { + barrier_var(sum_w); + sum_w += w_snap[i]; + } + + /* + * Split [0, nr_excl) into one contiguous range per participant, each + * the floor of its weight share. cp_range_owner[]/cp_range_end[] record + * each range's owner and cumulative end, total_excl counts the + * exclusive slots, and the rest (nr_excl - total_excl) are shared. + * rr_slots[] lists every participant for the round-robin. + */ + share = (u64)nr_excl * self_w / sum_w; + total_excl += share; + cp_range_owner[0] = CID_SELF; + cp_range_end[0] = total_excl; + qa.part.rr_slots[nr_rr++] = 0; /* self holds slot 0 (cgid 0 = no grant) */ + + bpf_for(i, 0, MAX_SUB_SCHEDS) { + u64 cgid = cgid_snap[i]; + s32 w = w_snap[i]; + + barrier_var(total_excl); + share = (u64)nr_excl * w / sum_w; + total_excl += share; + cp_range_owner[i + 1] = cgid ? i : CID_NONE; + cp_range_end[i + 1] = total_excl; + + if (cgid) { + barrier_var(nr_rr); + if (nr_rr < 0 || nr_rr >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + qa.part.rr_slots[nr_rr++] = cgid; + } + } + + /* assign each cid: held-excl by position, the rest self/none */ + bpf_for(cid, 0, nr_cids) { + if (cmask_test(cid, &qa.held_excl.mask)) { + place_one(cid, n, total_excl); + n++; + barrier_var(n); + } else if (cmask_test(cid, &qa.held_shared.mask)) { + qa.part.cid_owner[cid] = CID_SELF; /* time-share, self-local */ + } else { + qa.part.cid_owner[cid] = CID_NONE; /* not held */ + } + } + + qa.part.nr_shared = nr_excl - total_excl; + qa.part.nr_rr = nr_rr; +} + +/* + * Charge elapsed wall time to each cid's current owner. Runs under the + * partition guard before every ownership change and from the stats flush, so + * alloc_ns[] reflects the layout that was in effect. Shared-pool time is + * charged to the live round-robin holder. + */ +static __noinline void account_alloc(void) +{ + u64 now = bpf_ktime_get_ns(); + s32 rr_owner = CID_SELF; + s32 nr_cids = qa.nr_cids; + u64 delta; + s32 cid, i; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* first call starts the clock */ + if (!qa.alloc_ts) { + qa.alloc_ts = now; + return; + } + delta = now - qa.alloc_ts; + qa.alloc_ts = now; + qa.alloc_window_ns += delta; + + /* resolve the live shared-pool holder to an owner id */ + if (qa.part.nr_shared && qa.part.nr_rr) { + u32 pos = qa.part.rr_pos; + u64 cgid = pos < MAX_PARTS ? qa.part.rr_slots[pos] : 0; + + if (cgid) { + rr_owner = CID_NONE; + bpf_for(i, 0, MAX_SUB_SCHEDS) + if (qa.sub_sched_ctxs[i].cgroup_id == cgid) + rr_owner = i; + } + } + + bpf_for(cid, 0, nr_cids) { + s32 owner = qa.part.cid_owner[cid]; + + if (owner == CID_SHARED) + owner = rr_owner; + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + qa.alloc_ns[owner] += delta; + else if (owner == CID_SELF) + qa.self_alloc_ns += delta; + } +} + +/* + * apply_partition - execute the plan compute_partition() built + * + * Turn the owner map into the per-child, shared and self cmasks and issue the + * grant/revoke kfuncs as a delta against each child's previous grant. If no + * excl cid, evict every child. + */ +__noinline void apply_partition(void) +{ + s32 nr_cids = qa.nr_cids; + s32 nr_shared = qa.part.nr_shared; + s32 i, cid; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || + nr_shared < 0 || nr_shared > MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* no excl cpu: run own tasks on the held shares, evict children */ + if (!qa.part.nr_excl) { + cmask_copy(&qa.self_cids.mask, &qa.held_shared.mask); + bpf_for(i, 0, MAX_SUB_SCHEDS) + if (qa.sub_sched_ctxs[i].cgroup_id) + scx_bpf_sub_kill(qa.sub_sched_ctxs[i].cgroup_id, + "parent holds no excl cpu to distribute"); + return; + } + + /* + * Snapshot the old pool. The per-child revoke below clears ENQ_IMMED on + * the previously-granted pool, so a cid that left the pool (now a + * sibling's excl) doesn't keep a stale ENQ_IMMED on its last holder. + */ + cmask_copy(&qa.prev_rr_cids.mask, &qa.rr_cids.mask); + + /* turn the owner map into the rr pool, per-child excl, and self sets */ + cmask_init(&qa.rr_cids.mask, 0, nr_cids); + cmask_init(&qa.self_cids.mask, 0, nr_cids); + + /* snapshot each child's grant, then rebuild the new sets below */ + bpf_for(i, 0, MAX_SUB_SCHEDS) { + cmask_copy(&qa.sub_sched_ctxs[i].prev_granted.mask, + &qa.sub_sched_ctxs[i].granted_cids.mask); + cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids); + } + + bpf_for(i, 0, nr_shared) + cmask_set(qa.part.shared_cids[i], &qa.rr_cids.mask); + bpf_for(cid, 0, nr_cids) { + s32 o = qa.part.cid_owner[cid]; + + if (cmask_test(cid, &qa.rr_cids.mask)) + continue; + if (o >= 0 && o < MAX_SUB_SCHEDS) + cmask_set(cid, &qa.sub_sched_ctxs[o].granted_cids.mask); + else if (o == CID_SELF) + cmask_set(cid, &qa.self_cids.mask); + } + + /* + * Apply each child's exclusive cids as a delta against its previous + * grant. Separately clear the previous shared grant (ENQ_IMMED on the + * old pool), covering cids still pooled and cids that left for a + * sibling's excl. The current holder is granted the new pool below. + */ + bpf_for(i, 0, MAX_SUB_SCHEDS) { + struct sub_sched_ctx __arena *ssc = &qa.sub_sched_ctxs[i]; + u64 cgid = ssc->cgroup_id; + + if (!cgid) + continue; + + cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask); + cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask); + cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask); + cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); + + scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.prev_rr_cids.mask); + scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.to_revoke_cids.mask); + scx_bpf_sub_grant(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.to_grant_cids.mask, NULL); + } + + /* the current holder of the shared pool gets ENQ_IMMED on all of it */ + if (nr_shared) { + s32 pos = qa.part.rr_pos; + u64 holder_cgid; + + if (pos < 0 || pos >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + holder_cgid = qa.part.rr_slots[pos]; /* 0 = self, nothing to grant */ + if (holder_cgid) + scx_bpf_sub_grant(holder_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.rr_cids.mask, NULL); + } +} + +/* + * Recompute the split off the node's held caps and apply it. The contexts this + * runs from (the sub-sched and cgroup callbacks, the rr timer) are not + * serialized by the kernel, so a single runner does the work. A caller that + * finds the guard held leaves part_pending set; the holder drains it before + * releasing, with the rr timer as a backstop. + */ +static void redistribute(void) +{ + s32 i; + + __sync_fetch_and_or(&part_pending, 1); + + if (!part_try_start()) + return; + + bpf_for(i, 0, 1024) { + __sync_fetch_and_and(&part_pending, 0); + /* charge elapsed time to the current partition before rebuilding it */ + account_alloc(); + compute_partition(); + apply_partition(); + if (!__sync_fetch_and_or(&part_pending, 0)) + break; + } + + part_end(); +} + +/* + * Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading + * it for the stats display. Skipping when the partition guard is held is + * fine - alloc_ts is untouched, so the elapsed time is charged next time. + */ +SEC("syscall") +int flush_alloc(void *ctx) +{ + if (part_try_start()) { + account_alloc(); + part_end(); + } + return 0; +} + +/* + * Hand the shared pool to the next participant in the rotation. Self's turn + * just revokes the pool back to this sched. A child's turn grants it ENQ_IMMED + * on the entire pool. As only excl-held cids are time-shared, a wall-clock + * rotation works. Driven by the round-robin timer. + */ +static void rr_advance(void) +{ + s32 nr_shared, old_pos, new_pos; + u64 old_cgid, new_cgid; + u32 nr_rr; /* unsigned for % */ + + /* a redistribute holds the partition and rebuilds the pool, so skip */ + if (!part_try_start()) + return; + + nr_rr = qa.part.nr_rr; + nr_shared = qa.part.nr_shared; + + if (nr_shared < 0 || nr_shared > MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + if (nr_shared && nr_rr >= 2) { + /* close out the outgoing holder's pool time */ + account_alloc(); + + old_pos = qa.part.rr_pos; + new_pos = (old_pos + 1) % nr_rr; + old_cgid = qa.part.rr_slots[old_pos]; + new_cgid = qa.part.rr_slots[new_pos]; + qa.part.rr_pos = new_pos; + + /* + * Move the ENQ_IMMED cap to the next participant. The shared + * cids stay marked CID_SHARED. qmap_dispatch() resolves the + * live holder via rr_pos without the guard, so a dispatch + * racing this handoff may reenqueue a task once. Harmless for a + * time-share. + */ + if (old_cgid) + scx_bpf_sub_revoke(old_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.rr_cids.mask); + if (new_cgid) + scx_bpf_sub_grant(new_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, + &qa.rr_cids.mask, NULL); + } + + part_end(); + + /* a resplit queued while we held the guard supersedes this rotation */ + if (__sync_fetch_and_or(&part_pending, 0)) + redistribute(); +} + +/* advance the time-shared cid pool every round_robin_ns */ +static int round_robin_timerfn(void *map, int *key, struct bpf_timer *timer) +{ + rr_advance(); + bpf_timer_start(timer, round_robin_ns, 0); + return 0; +} + +/* + * Custom cid layout for the cid-override test. On invalid input the kfunc + * scx_error()s and aborts the scheduler. + */ +s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_cids) +{ + u32 nr_cpu_ids = scx_bpf_nr_cpu_ids(); + + if (!cid_override_mode) + return 0; + + /* the arena arrays are sized SCX_QMAP_MAX_CPUS */ + if (nr_cpu_ids > SCX_QMAP_MAX_CPUS) { + scx_bpf_error("nr_cpu_ids=%u exceeds SCX_QMAP_MAX_CPUS=%d", + nr_cpu_ids, SCX_QMAP_MAX_CPUS); + return -EINVAL; + } + + scx_bpf_cid_override(qa.cid_override_cpu_to_cid, nr_cpu_ids, + qa.cid_override_shard_start, cid_override_nr_shards); + return 0; +} + s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) { u8 __arena *slab; @@ -1089,16 +1796,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) return -EINVAL; } - /* - * cid-override test hook. Must run before anything that reads the - * cid space (scx_bpf_nr_cids, cmask_init, etc.). On invalid input, - * the kfunc calls scx_error() which aborts the scheduler. - */ - if (cid_override_mode) { - scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid, - nr_cpu_ids * sizeof(s32)); - } - /* * Allocate the task_ctx slab in arena and thread the entire slab onto * the free list. max_tasks is set by userspace before load. Each entry @@ -1129,16 +1826,43 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) } qa.task_free_head = (task_ctx_t *)slab; - /* - * Allocate and initialize the idle cmask. Starts empty - update_idle - * fills it as cpus enter idle. - */ - qa_idle_cids = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); - if (!qa_idle_cids) { - scx_bpf_error("failed to allocate idle cmask"); - return -ENOMEM; + /* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */ + qa.nr_cids = nr_cids; + + /* cmasks are embedded in qa, so they only need initializing */ + cmask_init(&qa.idle_cids.mask, 0, nr_cids); + cmask_init(&qa.rr_cids.mask, 0, nr_cids); + cmask_init(&qa.prev_rr_cids.mask, 0, nr_cids); + cmask_init(&qa.self_cids.mask, 0, nr_cids); + cmask_init(&qa.to_revoke_cids.mask, 0, nr_cids); + cmask_init(&qa.to_grant_cids.mask, 0, nr_cids); + cmask_init(&qa.held_excl.mask, 0, nr_cids); + cmask_init(&qa.held_shared.mask, 0, nr_cids); + + scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); + cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); + + bpf_for(i, 0, MAX_SUB_SCHEDS) { + cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids); + cmask_init(&qa.sub_sched_ctxs[i].prev_granted.mask, 0, nr_cids); } - cmask_init(qa_idle_cids, 0, nr_cids); + + /* + * The root starts holding every cid. qmap_sub_ecaps_updated() maintains + * per-cid shared state as effective caps settle, and redistribute() + * rebuilds owner and self from held caps. A non-root node starts with + * nothing. + */ + bpf_for(i, 0, nr_cids) { + if (!sub_cgroup_id) { + cmask_set(i, &qa.self_cids.mask); + qa.part.cid_owner[i] = CID_SELF; + } else { + qa.part.cid_owner[i] = CID_NONE; + } + } + qa.part.nr_shared = 0; ret = scx_bpf_create_dsq(SHARED_DSQ, -1); if (ret) { @@ -1177,6 +1901,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) return ret; } + /* sub-sched: drive the boundary-cid round-robin from a bpf timer */ + timer = bpf_map_lookup_elem(&round_robin_timer, &key); + if (!timer) + return -ESRCH; + bpf_timer_init(timer, &round_robin_timer, CLOCK_MONOTONIC); + bpf_timer_set_callback(timer, round_robin_timerfn); + ret = bpf_timer_start(timer, round_robin_ns, 0); + if (ret) + return ret; + return 0; } @@ -1185,17 +1919,51 @@ void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) UEI_RECORD(uei, ei); } +/* + * Seed a new sub slot with the cgroup's current weight. The kernel delivers + * ops.cpuctl_set_weight() only on value-changing writes, so a weight set + * before the sub attached would otherwise go unnoticed. + */ +static u32 cgrp_cur_weight(u64 cgid) +{ + struct cgroup_subsys_state *css; + struct cgroup *cgrp; + u32 weight = 100; + + cgrp = bpf_cgroup_from_id(cgid); + if (!cgrp) + return weight; + + css = BPF_CORE_READ(cgrp, subsys[cpu_cgrp_id]); + if (css) { + struct task_group *tg = container_of(css, struct task_group, css); + u32 w = BPF_CORE_READ(tg, scx.weight); + + if (w) + weight = w; + } + bpf_cgroup_release(cgrp); + return weight; +} + s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) { s32 i; + /* as long as there is at least one excl cpu, children can attach */ + if (!cmask_weight(&qa.held_excl.mask)) + return -ENOSPC; + for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (!qa.sub_sched_cgroup_ids[i]) { - qa.sub_sched_cgroup_ids[i] = args->ops->sub_cgroup_id; - bpf_printk("attaching sub-sched[%d] on %s", - i, args->cgroup_path); - return 0; - } + if (qa.sub_sched_ctxs[i].cgroup_id) + continue; + + qa.sub_sched_ctxs[i].cgroup_id = args->ops->sub_cgroup_id; + qa.sub_sched_ctxs[i].weight = cgrp_cur_weight(args->ops->sub_cgroup_id); + qa.nr_sub_scheds++; + bpf_printk("attaching sub-sched[%d] on %s", i, args->cgroup_path); + redistribute(); + return 0; } return -ENOSPC; @@ -1206,15 +1974,37 @@ void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) s32 i; for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (qa.sub_sched_cgroup_ids[i] == args->ops->sub_cgroup_id) { - qa.sub_sched_cgroup_ids[i] = 0; - bpf_printk("detaching sub-sched[%d] on %s", - i, args->cgroup_path); - break; - } + if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id) + continue; + + qa.sub_sched_ctxs[i].cgroup_id = 0; + qa.sub_sched_ctxs[i].weight = 100; + cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids); + qa.nr_sub_scheds--; + bpf_printk("detaching sub-sched[%d] on %s", i, args->cgroup_path); + redistribute(); + break; } } +void BPF_STRUCT_OPS(qmap_sub_caps_updated, const struct scx_cmask *cmask, u64 caps) +{ + /* our held caps changed, redistribute */ + redistribute(); +} + +void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) +{ + /* + * Effective caps updated. Track which cids hold shared caps so a self + * task placed there enqueues IMMED. + */ + if (after & SCX_CAP_ENQ_IMMED) + qa.cid_shared[cid] = (after & SCX_CAP_ENQ) ? 0 : 1; + else + qa.cid_shared[cid] = 0; +} + SCX_OPS_CID_DEFINE(qmap_ops, .flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK, .select_cid = (void *)qmap_select_cid, @@ -1230,11 +2020,15 @@ SCX_OPS_CID_DEFINE(qmap_ops, .dump = (void *)qmap_dump, .dump_cid = (void *)qmap_dump_cid, .dump_task = (void *)qmap_dump_task, - .cgroup_init = (void *)qmap_cgroup_init, - .cgroup_set_weight = (void *)qmap_cgroup_set_weight, - .cgroup_set_bandwidth = (void *)qmap_cgroup_set_bandwidth, + .cpuctl_init = (void *)qmap_cpuctl_init, + .cpuctl_set_weight = (void *)qmap_cpuctl_set_weight, + .cpuctl_set_bandwidth = (void *)qmap_cpuctl_set_bandwidth, + .cpuctl_move = (void *)qmap_cpuctl_move, .sub_attach = (void *)qmap_sub_attach, .sub_detach = (void *)qmap_sub_detach, + .sub_caps_updated = (void *)qmap_sub_caps_updated, + .sub_ecaps_updated = (void *)qmap_sub_ecaps_updated, + .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, .timeout_ms = 5000U, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 67ddd483a4c7..5bb5f687e579 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -4,6 +4,9 @@ * Copyright (c) 2022 Tejun Heo * Copyright (c) 2022 David Vernet */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include #include #include @@ -12,6 +15,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include "scx_qmap.h" @@ -20,11 +27,27 @@ const char help_fmt[] = "A simple five-level FIFO queue sched_ext scheduler.\n" "\n" -"See the top-level comment in .bpf.c for more details.\n" +"It also demonstrates hierarchical sub-scheduling: a scheduler can hand some\n" +"of its cpus to a child cgroup that runs its own scheduler. Run one qmap as\n" +"the parent, then run another qmap on a child cgroup with -c to attach it\n" +"beneath the parent.\n" +"\n" +"The policy below is deliberately simplistic and the resulting behavior can\n" +"look odd. qmap is a demo: it exists to exercise every sub-scheduling\n" +"primitive the kernel offers with as little code as possible, not to schedule\n" +"well.\n" +"\n" +"A parent divides the full cpus it holds among itself and its children in\n" +"proportion to cpu.weight. The cpus left over by rounding are time-shared,\n" +"handed to each participant in turn every -R ms. A cpu a scheduler only\n" +"holds a time-share of is never handed further down, and a parent left with\n" +"no full cpu of its own shuts its children down.\n" +"\n" +"See the top-of-file comment in .bpf.c for the design.\n" "\n" "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n" " [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n" -" [-F COUNT] [-v]\n" +" [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n" "\n" " -s SLICE_US Override slice duration\n" " -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" @@ -43,7 +66,14 @@ const char help_fmt[] = " -p Switch only tasks on SCHED_EXT policy instead of all\n" " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n" " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n" -" -C MODE cid-override test (shuffle|bad-dup|bad-range)\n" +" -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n" +" -i SEC Stats interval, seconds (default 5)\n" +" -R MS Round-robin period for time-shared cpus, ms (default 200)\n" +" -J MODE Fault injection (wrong-cid: dispatch to a cid not held,\n" +" init-fail/cgrp-init-fail: fail init_task/cpuctl_init for\n" +" \"qmfail*\" comms/cgroups)\n" +" -B PPT Rescue bandwidth in parts per thousand, 0 disables (root only, default 20)\n" +" -q US Rescue batch quantum in microseconds (root only, default 5000)\n" " -v Print libbpf debug messages\n" " -h Display this help and exit\n"; @@ -62,14 +92,157 @@ static void sigint_handler(int dummy) exit_req = 1; } +static void invoke_flush_alloc(struct scx_qmap *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + + bpf_prog_test_run_opts(bpf_program__fd(skel->progs.flush_alloc), &opts); +} + +/* previous counter snapshots for the per-interval hier stats */ +struct hier_prev { + u64 alloc_ns[MAX_SUB_SCHEDS]; + u64 self_alloc_ns; + u64 alloc_window_ns; + u64 nr_dsps[MAX_SUB_SCHEDS]; + u64 nr_reenq_cap; + u64 nr_reenq_immed; + u64 nr_inject_attempts; + u64 nr_rescue_dsp; +}; + +/* current wall-clock time as "HH:MM:SS" for the startup and interval headers */ +static const char *tstamp(char *buf, size_t sz) +{ + time_t now = time(NULL); + + strftime(buf, sz, "%H:%M:%S", localtime(&now)); + return buf; +} + +/* format the cids whose cid_owner[] matches @owner as "0-3,8", "-" if none */ +static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_t sz) +{ + u32 nr = qa->nr_cids, cid; + size_t off = 0; + s32 start = -1; + + buf[0] = '\0'; + for (cid = 0; cid <= nr; cid++) { + bool match = cid < nr && qa->part.cid_owner[cid] == owner; + int n; + + if (match) { + if (start < 0) + start = cid; + continue; + } + if (start < 0) + continue; + + if (start == (s32)cid - 1) + n = snprintf(buf + off, sz - off, "%s%d", + off ? "," : "", start); + else + n = snprintf(buf + off, sz - off, "%s%d-%d", + off ? "," : "", start, cid - 1); + if (n < 0 || (size_t)n >= sz - off) { + strcpy(&buf[sz - 4], "..."); + return; + } + off += n; + start = -1; + } + if (!off) + strcpy(buf, "-"); +} + +/* partition summary + one row per sched: weight, cpus, dispatch rate, cids */ +static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid) +{ + char ranges[128], who[16]; + const char *rr = "-"; + double secs; + u32 i; + + /* + * account_alloc() bumps alloc_window_ns together with the per-owner + * counters, so dividing by the same window yields exact cid counts. + */ + secs = (qa->alloc_window_ns - prev->alloc_window_ns) / 1e9; + prev->alloc_window_ns = qa->alloc_window_ns; + + /* resolve the live shared-pool holder */ + if (qa->part.nr_shared && qa->part.nr_rr) { + u64 cgid = qa->part.rr_slots[qa->part.rr_pos]; + + rr = "self"; + if (cgid) { + rr = "?"; + for (i = 0; i < MAX_SUB_SCHEDS; i++) { + if (qa->sub_sched_ctxs[i].cgroup_id == cgid) { + snprintf(who, sizeof(who), "sub%u", i); + rr = who; + break; + } + } + } + } + + format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges)); + printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n", + (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr, + (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap), + (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed), + (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts), + (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp)); + prev->nr_reenq_cap = qa->nr_reenq_cap; + prev->nr_reenq_immed = qa->nr_reenq_immed; + prev->nr_inject_attempts = qa->nr_inject_attempts; + prev->nr_rescue_dsp = qa->nr_rescue_dsp; + + printf("hier : %-4s %10s %4s %6s %8s %s\n", + "", "cgroup", "w", "alloc", "disp/s", "cids"); + + format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges)); + printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self", + (unsigned long long)own_cgid, 100, + secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0, + "-", ranges); + prev->self_alloc_ns = qa->self_alloc_ns; + + for (i = 0; i < MAX_SUB_SCHEDS; i++) { + struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i]; + + if (!sc->cgroup_id) + continue; + + snprintf(who, sizeof(who), "sub%u", i); + format_cid_ranges(qa, i, ranges, sizeof(ranges)); + printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who, + (unsigned long long)sc->cgroup_id, sc->weight, + secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0, + secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0, + ranges); + prev->alloc_ns[i] = qa->alloc_ns[i]; + prev->nr_dsps[i] = sc->nr_dsps; + } +} + int main(int argc, char **argv) { struct scx_qmap *skel; struct bpf_link *link; struct qmap_arena *qa; - __u32 test_error_cnt = 0; - __u64 ecode; - int opt; + u32 test_error_cnt = 0; + u64 ecode; + int opt, stats_intv = 5, i, round_robin_ms = 200; + struct hier_prev hprev = {}; + const char *sub_cg_path = NULL; + char tbuf[32]; + u32 inject_mode = 0; + u64 own_cgid = 0; + s32 cid_override_shard_sz = 4; libbpf_set_print(libbpf_print_fn); signal(SIGINT, sigint_handler); @@ -84,12 +257,13 @@ int main(int argc, char **argv) } restart: optind = 1; - skel = SCX_OPS_OPEN(qmap_ops, scx_qmap); + skel = SCX_OPS_CID_OPEN(qmap_ops, scx_qmap); skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL"); skel->rodata->max_tasks = 16384; - while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:vh")) != -1) { + while ((opt = getopt(argc, argv, + "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) { switch (opt) { case 's': skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; @@ -129,6 +303,8 @@ int main(int argc, char **argv) } skel->struct_ops.qmap_ops->sub_cgroup_id = st.st_ino; skel->rodata->sub_cgroup_id = st.st_ino; + own_cgid = st.st_ino; + sub_cg_path = optarg; break; } case 'd': @@ -146,7 +322,6 @@ int main(int argc, char **argv) skel->struct_ops.qmap_ops->flags |= SCX_OPS_SWITCH_PARTIAL; break; case 'I': - skel->rodata->always_enq_immed = true; skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED; break; case 'F': @@ -154,33 +329,73 @@ int main(int argc, char **argv) break; case 'C': { u32 nr_cpus = libbpf_num_possible_cpus(); - u32 mode, i; + u32 mode; if (!strcmp(optarg, "shuffle")) - mode = 1; + mode = QMAP_CID_OVR_SHUFFLE; else if (!strcmp(optarg, "bad-dup")) - mode = 2; + mode = QMAP_CID_OVR_BAD_DUP; else if (!strcmp(optarg, "bad-range")) - mode = 3; + mode = QMAP_CID_OVR_BAD_RANGE; + else if (!strcmp(optarg, "bad-mono")) + mode = QMAP_CID_OVR_BAD_MONO; else { fprintf(stderr, "unknown cid-override mode '%s'\n", optarg); return 1; } skel->rodata->cid_override_mode = mode; + cid_override_shard_sz = 4; - /* shuffle: reversed cpu_to_cid, bad-dup: dup cid 0, bad-range: identity */ - for (i = 0; i < nr_cpus; i++) { - if (mode == 1) - skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; - else - skel->bss->cid_override_cpu_to_cid[i] = i; + /* + * bad-mono needs >= 3 shards to build a 0-based but + * non-monotonic shard_start. Shrink the shard size so + * the test runs on any machine with >= 3 cpus. + */ + if (mode == QMAP_CID_OVR_BAD_MONO) { + if (nr_cpus < 3) { + fprintf(stderr, "bad-mono needs >= 3 cpus (have %u)\n", + nr_cpus); + return 1; + } + cid_override_shard_sz = nr_cpus / 3; } - if (mode == 2 && nr_cpus >= 2) - skel->bss->cid_override_cpu_to_cid[1] = 0; - if (mode == 3) - skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus; + + /* shards of shard_sz each */ + skel->rodata->cid_override_nr_shards = + (nr_cpus + cid_override_shard_sz - 1) / cid_override_shard_sz; break; } + case 'i': + stats_intv = atoi(optarg); + if (stats_intv < 1) + stats_intv = 1; + break; + case 'R': + round_robin_ms = atoi(optarg); + if (round_robin_ms < 10) + round_robin_ms = 10; + break; + case 'J': + if (!strcmp(optarg, "wrong-cid")) + inject_mode = QMAP_INJ_WRONG_CID; + else if (!strcmp(optarg, "init-fail")) + inject_mode = QMAP_INJ_INIT_FAIL; + else if (!strcmp(optarg, "cgrp-init-fail")) + inject_mode = QMAP_INJ_CGRP_INIT_FAIL; + else + inject_mode = strtoul(optarg, NULL, 0); + break; + case 'B': { + u32 ppt = strtoul(optarg, NULL, 0); + + if (!ppt) + ppt = __COMPAT_ENUM_OR_ZERO("scx_consts", "SCX_RESCUE_DISABLE"); + skel->struct_ops.qmap_ops->rescue_bandwidth_ppt = ppt; + break; + } + case 'q': + skel->struct_ops.qmap_ops->rescue_quantum_us = strtoul(optarg, NULL, 0); + break; case 'v': verbose = true; break; @@ -190,27 +405,75 @@ int main(int argc, char **argv) } } + skel->rodata->round_robin_ns = (u64)round_robin_ms * 1000000; + SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei); - link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap); qa = &skel->arena->qa; + + /* + * The cid-override arrays live in the arena, which is mmapped at load. + * Populate them before qmap_init_cids() consumes them at attach. + */ + if (skel->rodata->cid_override_mode) { + u32 mode = skel->rodata->cid_override_mode; + u32 nr_cpus = libbpf_num_possible_cpus(); + u32 i; + + /* shuffle: reversed cpu_to_cid; others: identity */ + for (i = 0; i < nr_cpus; i++) { + if (mode == QMAP_CID_OVR_SHUFFLE) + qa->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; + else + qa->cid_override_cpu_to_cid[i] = i; + } + if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2) + qa->cid_override_cpu_to_cid[1] = 0; + if (mode == QMAP_CID_OVR_BAD_RANGE) + qa->cid_override_cpu_to_cid[0] = (s32)nr_cpus; + + for (i = 0; i < skel->rodata->cid_override_nr_shards; i++) + qa->cid_override_shard_start[i] = i * cid_override_shard_sz; + + if (mode == QMAP_CID_OVR_BAD_MONO) { + /* swap [1] and [2] to break monotonicity */ + s32 tmp = qa->cid_override_shard_start[1]; + qa->cid_override_shard_start[1] = qa->cid_override_shard_start[2]; + qa->cid_override_shard_start[2] = tmp; + } + } + + link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap); + qa->test_error_cnt = test_error_cnt; + qa->inject_mode = inject_mode; + + if (sub_cg_path) + printf("%s scx_qmap started: sub-scheduler on %s, stats every %ds\n", + tstamp(tbuf, sizeof(tbuf)), sub_cg_path, stats_intv); + else + printf("%s scx_qmap started: root scheduler, stats every %ds\n", + tstamp(tbuf, sizeof(tbuf)), stats_intv); + fflush(stdout); while (!exit_req && !UEI_EXITED(skel, uei)) { long nr_enqueued = qa->nr_enqueued; long nr_dispatched = qa->nr_dispatched; + printf("---- %s ----\n", + tstamp(tbuf, sizeof(tbuf))); printf("stats : enq=%lu dsp=%lu delta=%ld reenq/cid0=%llu/%llu deq=%llu core=%llu enq_ddsp=%llu\n", nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, - qa->nr_reenqueued, qa->nr_reenqueued_cid0, - qa->nr_dequeued, - qa->nr_core_sched_execed, - qa->nr_ddsp_from_enq); + (unsigned long long)qa->nr_reenqueued, + (unsigned long long)qa->nr_reenqueued_cid0, + (unsigned long long)qa->nr_dequeued, + (unsigned long long)qa->nr_core_sched_execed, + (unsigned long long)qa->nr_ddsp_from_enq); printf(" exp_local=%llu exp_remote=%llu exp_timer=%llu exp_lost=%llu\n", - qa->nr_expedited_local, - qa->nr_expedited_remote, - qa->nr_expedited_from_timer, - qa->nr_expedited_lost); + (unsigned long long)qa->nr_expedited_local, + (unsigned long long)qa->nr_expedited_remote, + (unsigned long long)qa->nr_expedited_from_timer, + (unsigned long long)qa->nr_expedited_lost); if (__COMPAT_has_ksym("scx_bpf_cidperf_cur")) printf("cpuperf: cur min/avg/max=%u/%u/%u target min/avg/max=%u/%u/%u\n", qa->cpuperf_min, @@ -219,15 +482,20 @@ int main(int argc, char **argv) qa->cpuperf_target_min, qa->cpuperf_target_avg, qa->cpuperf_target_max); + + invoke_flush_alloc(skel); + print_hier(qa, &hprev, own_cgid); fflush(stdout); - sleep(1); + + for (i = 0; i < stats_intv && !exit_req && !UEI_EXITED(skel, uei); i++) + sleep(1); } bpf_link__destroy(link); ecode = UEI_REPORT(skel, uei); scx_qmap__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index d15a705d5ac5..c78d61806b39 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -20,6 +20,7 @@ #endif #define MAX_SUB_SCHEDS 8 +#define MAX_PARTS (MAX_SUB_SCHEDS + 1) /* participants: children + self */ /* * cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and @@ -27,47 +28,159 @@ */ #define SCX_QMAP_MAX_CPUS 1024 +/* + * An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS), + * CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED + * for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used + * by the partition's cid_owner[]. + */ +#define CID_SELF (-1) +#define CID_NONE (-2) +#define CID_SHARED (-3) + +/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */ +enum qmap_cid_override { + QMAP_CID_OVR_OFF = 0, /* disabled */ + QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */ + QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */ + QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */ + QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */ +}; + struct cpu_ctx { - __u64 dsp_idx; /* dispatch index */ - __u64 dsp_cnt; /* remaining count */ - __u32 avg_weight; - __u32 cpuperf_target; + u64 dsp_idx; /* dispatch index */ + u64 dsp_cnt; /* remaining count */ + u32 avg_weight; + u32 cpuperf_target; +}; + +struct qmap_fifo { + struct task_ctx __arena *head; + struct task_ctx __arena *tail; + s32 idx; +}; + +/* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */ +enum qmap_inject { + QMAP_INJ_OFF = 0, + QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */ + QMAP_INJ_INIT_FAIL = 2, /* fail init_task for "qmfail*" comms */ + QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cpuctl_init for "qmfail*" cgroups */ +}; + +/* + * scx_cmask's are embedded in struct qmap_arena with inline backing storage. + * The bpf side uses &field.mask with the normal cmask_* helpers. Userspace + * doesn't have access to the type definition and sees same-sized opaque words. + * _Static_assert()'s in .bpf.c ensure that they are in sync. + */ +#define QMAP_CMASK_WORDS (((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1) +struct qmap_cmask { +#ifdef __BPF__ + union { + struct scx_cmask mask; + u64 words[QMAP_CMASK_WORDS + 2]; + }; +#else + u64 words[QMAP_CMASK_WORDS + 2]; +#endif }; /* Opaque to userspace; defined in scx_qmap.bpf.c. */ struct task_ctx; -struct qmap_fifo { - struct task_ctx __arena *head; - struct task_ctx __arena *tail; - __s32 idx; +/* per-direct-child state for the sub-scheduler */ +struct sub_sched_ctx { + u64 cgroup_id; + u32 weight; /* cpu.weight, seeded at attach, then set_weight */ + u64 nr_dsps; + struct qmap_cmask granted_cids; /* cids granted excl to this child */ + struct qmap_cmask prev_granted; /* last grant, for delta calculation */ +}; + +/* + * compute_partition() builds the following from this node's held caps, and + * apply_partition()/rr_advance() execute it. Userspace only reads for the + * hierarchy display. + */ +struct qmap_partition { + u32 nr_excl; /* number of excl-held (delegatable) cids */ + s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */ + s32 shared_cids[MAX_PARTS]; /* the round-robin cid pool */ + u32 nr_shared; /* number of shared_cids entries */ + u64 rr_slots[MAX_PARTS]; /* rotation order: holder cgroup_id, 0 = self */ + u32 nr_rr; /* number of rr_slots entries */ + u32 rr_pos; /* current rotation index */ }; struct qmap_arena { /* userspace-visible stats */ - __u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0; - __u64 nr_dequeued, nr_ddsp_from_enq; - __u64 nr_core_sched_execed; - __u64 nr_expedited_local, nr_expedited_remote; - __u64 nr_expedited_lost, nr_expedited_from_timer; - __u64 nr_highpri_queued; - __u32 test_error_cnt; - __u32 cpuperf_min, cpuperf_avg, cpuperf_max; - __u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max; + u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0; + u64 nr_dequeued, nr_ddsp_from_enq; + u64 nr_core_sched_execed; + u64 nr_expedited_local, nr_expedited_remote; + u64 nr_expedited_lost, nr_expedited_from_timer; + u64 nr_highpri_queued; + u32 test_error_cnt; + u32 cpuperf_min, cpuperf_avg, cpuperf_max; + u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max; /* kernel-side runtime state */ - __u64 sub_sched_cgroup_ids[MAX_SUB_SCHEDS]; - __u64 core_sched_head_seqs[5]; - __u64 core_sched_tail_seqs[5]; + u64 core_sched_head_seqs[5]; + u64 core_sched_tail_seqs[5]; struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS]; + /* cid-override test input, populated by the loader before attach */ + __s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; + __s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS]; + /* task_ctx slab; allocated and threaded by qmap_init() */ struct task_ctx __arena *task_ctxs; struct task_ctx __arena *task_free_head; /* five priority FIFOs, each a doubly-linked list through task_ctx */ struct qmap_fifo fifos[5]; + + /* + * Hierarchical sub-scheduling state. See the design comment at the top + * of scx_qmap.bpf.c. + */ + u32 nr_cids; /* cid count, cached at init */ + + /* bpf-owned partition: read by userspace for display */ + struct qmap_partition part; + + struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */ + u64 nr_sub_scheds; /* number of attached children */ + + /* bpf-internal per-cid state */ + u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */ + + /* allocated cid-time, charged per owner by account_alloc() */ + u64 alloc_ns[MAX_SUB_SCHEDS]; /* per child slot */ + u64 self_alloc_ns; + u64 alloc_ts; /* last accounting timestamp */ + u64 alloc_window_ns; /* total accounted time, the alloc denominator */ + + /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ + struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ + struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */ + struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */ + + /* scratch cmasks */ + struct qmap_cmask to_revoke_cids; /* delta cids to revoke */ + struct qmap_cmask to_grant_cids; /* delta cids to grant */ + struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */ + struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */ + struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */ + + /* bpf -> userspace: stats */ + u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ + u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ + u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ + u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ + u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ }; #endif /* __SCX_QMAP_H */ diff --git a/tools/sched_ext/scx_sdt.c b/tools/sched_ext/scx_sdt.c index bf664b2d3785..2f93a00de548 100644 --- a/tools/sched_ext/scx_sdt.c +++ b/tools/sched_ext/scx_sdt.c @@ -44,7 +44,7 @@ int main(int argc, char **argv) { struct scx_sdt *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); @@ -96,7 +96,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_sdt__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_simple.c b/tools/sched_ext/scx_simple.c index c3b48611712b..b7589a83f28a 100644 --- a/tools/sched_ext/scx_simple.c +++ b/tools/sched_ext/scx_simple.c @@ -64,7 +64,7 @@ int main(int argc, char **argv) { struct scx_simple *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); @@ -104,7 +104,7 @@ int main(int argc, char **argv) ecode = UEI_REPORT(skel, uei); scx_simple__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/sched_ext/scx_userland.c b/tools/sched_ext/scx_userland.c index 616043c165e6..b16b6db5f365 100644 --- a/tools/sched_ext/scx_userland.c +++ b/tools/sched_ext/scx_userland.c @@ -52,6 +52,7 @@ static __u32 batch_size = 8; static bool verbose; static volatile int exit_req; +static volatile int stats_stop; static int enqueued_fd, dispatched_fd; static pthread_t stats_printer; @@ -286,7 +287,7 @@ static void dispatch_batch(void) static void *run_stats_printer(void *arg) { - while (!exit_req) { + while (!stats_stop) { __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; nr_failed_enqueues = skel->bss->nr_failed_enqueues; @@ -326,7 +327,7 @@ static int spawn_stats_thread(void) static void pre_bootstrap(int argc, char **argv) { int err; - __u32 opt; + __s32 opt; struct sched_param sched_param = { .sched_priority = sched_get_priority_max(SCHED_EXT), }; @@ -374,7 +375,7 @@ static void pre_bootstrap(int argc, char **argv) static void bootstrap(char *comm) { - exit_req = 0; + stats_stop = 0; min_vruntime = 0.0; __atomic_store_n(&nr_vruntime_enqueues, 0, __ATOMIC_RELAXED); __atomic_store_n(&nr_vruntime_dispatches, 0, __ATOMIC_RELAXED); @@ -404,7 +405,7 @@ static void bootstrap(char *comm) static void sched_main_loop(void) { - while (!exit_req) { + while (!exit_req && !UEI_EXITED(skel, uei)) { /* * Perform the following work in the main user space scheduler * loop: @@ -434,13 +435,13 @@ int main(int argc, char **argv) bootstrap(argv[0]); sched_main_loop(); - exit_req = 1; + stats_stop = 1; bpf_link__destroy(ops_link); pthread_join(stats_printer, NULL); ecode = UEI_REPORT(skel, uei); scx_userland__destroy(skel); - if (UEI_ECODE_RESTART(ecode)) + if (!exit_req && UEI_ECODE_RESTART(ecode)) goto restart; return 0; } diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c index 35923e74a2ec..9dd72d0da29b 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c @@ -15,15 +15,48 @@ UEI_DEFINE(uei); private(PREF_CPUS) struct bpf_cpumask __kptr * allowed_cpumask; static void -validate_idle_cpu(const struct task_struct *p, const struct cpumask *allowed, s32 cpu) +validate_local_idle_state(void) { - if (scx_bpf_test_and_clear_cpu_idle(cpu)) - scx_bpf_error("CPU %d should be marked as busy", cpu); + const struct cpumask *idle; + struct task_struct *curr; + s32 cpu = bpf_get_smp_processor_id(); + bool cpu_is_idle, curr_is_idle; - if (bpf_cpumask_subset(allowed, p->cpus_ptr) && - !bpf_cpumask_test_cpu(cpu, allowed)) + bpf_rcu_read_lock(); + curr = scx_bpf_cpu_curr(cpu); + curr_is_idle = curr && (curr->flags & PF_IDLE); + bpf_rcu_read_unlock(); + + idle = scx_bpf_get_idle_cpumask(); + cpu_is_idle = bpf_cpumask_test_cpu(cpu, idle); + scx_bpf_put_idle_cpumask(idle); + + /* + * Unlike a remote selected CPU, the local CPU cannot go through an + * idle re-pick while this callback is running. If it is running a + * non-idle scheduling context, it must not be advertised as idle. + */ + if (!curr_is_idle && cpu_is_idle) + scx_bpf_error("running CPU %d should be marked as busy", cpu); +} + +static void +validate_selected_cpu(const struct task_struct *p, s32 cpu) +{ + const struct cpumask *allowed = cast_mask(allowed_cpumask); + + if (!allowed) { + scx_bpf_error("allowed domain not initialized"); + return; + } + + if (!bpf_cpumask_test_cpu(cpu, allowed)) scx_bpf_error("CPU %d not in the allowed domain for %d (%s)", cpu, p->pid, p->comm); + + if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) + scx_bpf_error("CPU %d not in the affinity mask for %d (%s)", + cpu, p->pid, p->comm); } s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu, @@ -32,6 +65,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu, const struct cpumask *allowed; s32 cpu; + validate_local_idle_state(); allowed = cast_mask(allowed_cpumask); if (!allowed) { scx_bpf_error("allowed domain not initialized"); @@ -43,7 +77,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu, */ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, allowed, 0); if (cpu >= 0) { - validate_idle_cpu(p, allowed, cpu); + validate_selected_cpu(p, cpu); scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0); return cpu; @@ -59,6 +93,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags) scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); + validate_local_idle_state(); allowed = cast_mask(allowed_cpumask); if (!allowed) { scx_bpf_error("allowed domain not initialized"); @@ -71,7 +106,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags) */ cpu = scx_bpf_select_cpu_and(p, prev_cpu, 0, allowed, 0); if (cpu >= 0) { - validate_idle_cpu(p, allowed, cpu); + validate_selected_cpu(p, cpu); scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE); } } diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c index 6f4c3f5a1c5d..7ef9de7b27eb 100644 --- a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c @@ -14,18 +14,16 @@ s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* - * If we dispatch to a bogus DSQ that will fall back to the - * builtin global DSQ, we fail gracefully. - */ - scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } - - return prev_cpu; + /* + * If we dispatch to a bogus DSQ that will fall back to the + * builtin global DSQ, we fail gracefully. + */ + scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); + return cpu; } void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c index e4a55027778f..82dca4cdc0a6 100644 --- a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c @@ -14,15 +14,14 @@ s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ - scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } + /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ + scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); - return prev_cpu; + return cpu; } void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index b987611789d1..01b17092d5c8 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -31,6 +31,7 @@ static enum scx_test_status run(void *ctx) continue; skel = exit__open(); + SCX_FAIL_IF(!skel, "Failed to open"); SCX_ENUM_INIT(skel); skel->rodata->exit_point = tc; SCX_FAIL_IF(exit__load(skel), "Failed to load skel"); diff --git a/tools/testing/selftests/sched_ext/numa.bpf.c b/tools/testing/selftests/sched_ext/numa.bpf.c index 6b4515c28aa0..679b51d38089 100644 --- a/tools/testing/selftests/sched_ext/numa.bpf.c +++ b/tools/testing/selftests/sched_ext/numa.bpf.c @@ -19,16 +19,31 @@ UEI_DEFINE(uei); const volatile unsigned int __COMPAT_SCX_PICK_IDLE_IN_NODE; -static bool is_cpu_idle(s32 cpu, int node) +static void validate_local_idle_state(void) { const struct cpumask *idle_cpumask; - bool idle; + struct task_struct *curr; + s32 cpu = bpf_get_smp_processor_id(); + int node = __COMPAT_scx_bpf_cpu_node(cpu); + bool cpu_is_idle, curr_is_idle; + + bpf_rcu_read_lock(); + curr = scx_bpf_cpu_curr(cpu); + curr_is_idle = curr && (curr->flags & PF_IDLE); + bpf_rcu_read_unlock(); idle_cpumask = __COMPAT_scx_bpf_get_idle_cpumask_node(node); - idle = bpf_cpumask_test_cpu(cpu, idle_cpumask); + cpu_is_idle = bpf_cpumask_test_cpu(cpu, idle_cpumask); scx_bpf_put_cpumask(idle_cpumask); - return idle; + /* + * Unlike a remote picked CPU, the local CPU cannot go through an + * idle re-pick while this callback is running. If it is running a + * non-idle scheduling context, it must not be advertised as idle + * in its node's idle cpumask. + */ + if (!curr_is_idle && cpu_is_idle) + scx_bpf_error("running CPU %d should be marked as busy", cpu); } s32 BPF_STRUCT_OPS(numa_select_cpu, @@ -38,6 +53,8 @@ s32 BPF_STRUCT_OPS(numa_select_cpu, int node = __COMPAT_scx_bpf_cpu_node(task_cpu); s32 cpu; + validate_local_idle_state(); + /* * We could just use __COMPAT_scx_bpf_pick_any_cpu_node() here, * since it already tries to pick an idle CPU within the node @@ -59,9 +76,6 @@ s32 BPF_STRUCT_OPS(numa_select_cpu, if (cpu < 0 && !bpf_cpumask_test_cpu(task_cpu, p->cpus_ptr)) return prev_cpu; - if (is_cpu_idle(cpu, node)) - scx_bpf_error("CPU %d should be marked as busy", cpu); - if (__COMPAT_scx_bpf_cpu_node(cpu) != node) scx_bpf_error("CPU %d should be in node %d", cpu, node); diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 05974820ca69..1129ec2aaddc 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx) static enum scx_test_status run(void *ctx) { struct prog_run *skel = ctx; - struct bpf_link *link; + struct bpf_link *link = NULL; + enum scx_test_status status = SCX_TEST_PASS; int prog_fd, err = 0; prog_fd = bpf_program__fd(skel->progs.prog_run_syscall); @@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops); if (!link) { SCX_ERR("Failed to attach scheduler"); - close(prog_fd); - return SCX_TEST_FAIL; + status = SCX_TEST_FAIL; + goto out; } err = bpf_prog_test_run_opts(prog_fd, &topts); - SCX_EQ(err, 0); + if (err) { + SCX_ERR("BPF_PROG_RUN failed (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } /* Assumes uei.kind is written last */ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) sched_yield(); - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); - SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef); - close(prog_fd); - bpf_link__destroy(link); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { + SCX_ERR("Unexpected exit kind: %llu", + (unsigned long long)skel->data->uei.kind); + status = SCX_TEST_FAIL; + goto out; + } + if (skel->data->uei.exit_code != 0xdeadbeef) { + SCX_ERR("Unexpected exit code: %lld", + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + goto out; + } - return SCX_TEST_PASS; +out: + close(prog_fd); + if (link) + bpf_link__destroy(link); + + return status; } static void cleanup(void *ctx)