diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index efd635a99309..f3564581c317 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -107,7 +107,21 @@ enum scx_ent_flags { SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ /* - * Bits 8 to 10 are used to carry task state: + * The kernel owns the slice, currently the rescue grant. While set: + * + * - refill_task_slice_dfl() and expire_task_slice() leave the slice + * alone. + * - Reenqueue scans skip the task. + * - put_prev_task_scx() keeps a preempted task on the local DSQ. + * - Schedulers may shorten the slice, never extend it. + */ + SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */ + + /* + * The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so + * that the single-bit flags stay contiguous from bit 0. + * + * Bits 25 to 27 are used to carry task state: * * NONE ops.init_task() not called yet * INIT_BEGIN ops.init_task() in flight; see sched_ext_dead() @@ -116,7 +130,7 @@ enum scx_ent_flags { * ENABLED fully initialized and in sched_ext * DEAD terminal state set by sched_ext_dead() */ - SCX_TASK_STATE_SHIFT = 8, + SCX_TASK_STATE_SHIFT = 25, SCX_TASK_STATE_BITS = 3, SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT, @@ -128,7 +142,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 to 14 are used to carry reenqueue reason. In addition to + * Bits 28 to 30 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -139,7 +153,7 @@ enum scx_ent_flags { * CAP sub-sched cap miss, see p->scx.reenq_reason_* * PROXY proxy state prevented a remote DSQ transfer */ - SCX_TASK_REENQ_REASON_SHIFT = 12, + SCX_TASK_REENQ_REASON_SHIFT = 28, SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index a5330b760ecd..8e04a8494e11 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -309,6 +309,12 @@ s32 scx_cid_init(struct scx_sched *sch) * empty shard. */ cids_in_core = cpumask_weight(core_scratch); + if (cids_in_core > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "core with %u cpus exceeds max shard size %d", + cids_in_core, SCX_CID_SHARD_MAX_CPUS); + return -E2BIG; + } + max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0); if (cores_in_shard && (cores_in_shard >= max_cores || diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9bae826d69f9..f74ffb3b1aca 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -388,8 +388,12 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) +static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { + struct scx_sched *host = scx_exec_sched(sch, cpu_of(rq)); + struct scx_dispatch_q *dsq; + /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -400,8 +404,29 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * Whenever a sched starts bypassing, all runnable tasks in its subtree * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. + * + * A cpu's dispatch only descends into a host holding %SCX_CAP_BASE on + * it, so the bypass DSQ of a host without BASE here is never consumed. + * Send @p to rescue on this cpu instead, resolving against @host. + * scx_dispatch_enqueue() resolves a local DSQ again, against the + * bypassing @sch. %SCX_ENQ_IGNORE_CAPS makes that second resolution + * admit @p. + * + * A migration-disabled task can't leave this cpu, and a task with a + * migration pending must run here until the migration moves it. Either + * would wait on the unconsumed bypass DSQ until the host regains BASE. + * Resolve them too when rescue is disabled. The resolution admits both + * to the local DSQ, as it does any insert of such a task. */ - return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); + if (unlikely(scx_missing_caps(host, cpu_of(rq), SCX_CAP_BASE)) && + (scx_rescue_enabled() || is_migration_disabled(p) || p->migration_pending)) { + *enq_flags |= SCX_ENQ_RESCUE; + dsq = scx_resolve_local_dsq(host, rq, p, enq_flags); + *enq_flags |= SCX_ENQ_IGNORE_CAPS; + } else { + dsq = scx_bypass_dsq(host, cpu_of(rq)); + } + return dsq; } /** @@ -1129,9 +1154,10 @@ static void schedule_deferred_locked(struct rq *rq) return; /* - * If in dispatch, the balance callbacks will be called before rq lock - * is released. Schedule one. - * + * If in dispatch without proxy execution, the balance callbacks will + * be called before rq lock is released. Schedule one. With proxy + * execution, proxy resolution can zap the callbacks without another + * SCX dispatch to requeue them, so use irq_work instead. * * We can't directly insert the callback into the * rq's list: The call can drop its lock and make the pending balance @@ -1139,7 +1165,7 @@ static void schedule_deferred_locked(struct rq *rq) * * Just let dispatch_one() know that it must do it itself. */ - if (rq->scx.flags & SCX_RQ_IN_DISPATCH) { + if ((rq->scx.flags & SCX_RQ_IN_DISPATCH) && !sched_proxy_exec()) { rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; return; } @@ -1152,6 +1178,36 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +static void resched_irq_workfn(struct irq_work *irq_work) +{ + struct rq *rq = container_of(irq_work, struct rq, scx.resched_irq_work); + + guard(rq_lock)(rq); + /* a higher class task would only be picked again, see kick_one_cpu() */ + if (!sched_class_above(rq->donor->sched_class, &ext_sched_class)) + resched_curr(rq); + scx_rq_lock_drop(rq); +} + +/** + * scx_resched_curr_deferred - Reschedule @rq's cpu from hard irq work on it + * @rq: rq to reschedule + * + * A kick, like a direct resched, only guarantees that a scheduling event on the + * cpu completes after it, and that can be the one in progress. Work that a pick + * processes only at the start of its dispatch, such as the ecaps syncs, can be + * queued after that point and miss the pick. A pick on @rq's cpu runs with IRQs + * off, so the irq work runs after it, and the scheduling event its resched + * triggers starts after this call. A sibling's core scheduling pick for @rq + * isn't covered. A cpu running a task of a higher class isn't rescheduled, as + * its pick would return that task again. @rq's cpu must stay online until the + * irq work is queued. + */ +void scx_resched_curr_deferred(struct rq *rq) +{ + irq_work_queue_on(&rq->scx.resched_irq_work, cpu_of(rq)); +} + #ifdef CONFIG_NO_HZ_FULL static void scx_proxy_tick_bal_cb(struct rq *rq) { @@ -1354,7 +1410,8 @@ enum scx_slice_oob_consts { * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is - * refused. See set_task_slice_keep_oob(). + * refused. See set_task_slice_keep_oob(). A kernel slice, see + * %SCX_TASK_KERNEL_SLICE, can be shortened but not extended. * * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct @@ -1441,8 +1498,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * @rq: rq @p is on * @p: task of interest * - * End what rides on the slice - the protection, and the rescue if @p is being - * rescued. + * End what rides on the slice - the protection, the kernel's ownership of the + * slice, and the rescue if @p is being rescued. * * A dequeue normally ends the slice too. The exception is a save/restore pair * on the running task. Attribute changes like renice cycle the task through @@ -1454,7 +1511,7 @@ void scx_task_slice_ended(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); - p->scx.flags &= ~SCX_TASK_PROTECTED; + p->scx.flags &= ~(SCX_TASK_PROTECTED | SCX_TASK_KERNEL_SLICE); if (unlikely(p == scx_rescuee(rq))) scx_rescue_end(rq); } @@ -1480,9 +1537,9 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 /* * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs - * baseline cpu access on @p's cid, shortening is always allowed, and a - * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See - * the write rules above. + * baseline cpu access on @p's cid and is refused on a kernel slice. Shortening + * is always allowed. A protected slice refuses both. %SCX_EV_SLICE_DENIED + * counts the denials. See the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1506,7 +1563,8 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; if (slice > p->scx.slice && - unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { + unlikely(scx_task_has_kernel_slice(p) || + scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); return; } @@ -1538,6 +1596,35 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e p->scx.dsq_vtime = vtime; } +/* + * refill_task_slice_dfl() supplies a slice when the scheduler left none and + * expire_task_slice() takes it away to force the task off the cpu. Neither + * touches a kernel slice, see %SCX_TASK_KERNEL_SLICE. + */ +static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) +{ + if (scx_task_has_kernel_slice(p)) + return; + + /* + * A default refill is not an explicit request, so it must not drop a + * pending out-of-band one, which is applied when @p next runs. + */ + set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); +} + +static void expire_task_slice(struct task_struct *p) +{ + /* + * Unlike the refill above, an expiry supersedes a pending out-of-band + * request. A kept request would be applied at the next + * update_curr_scx() and undo the expiry. + */ + if (!scx_task_has_kernel_slice(p)) + scx_set_task_slice(p, 0); +} + static void update_curr_scx(struct rq *rq) { struct task_struct *donor; @@ -1631,16 +1718,6 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) } } -static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) -{ - /* - * A default refill is not an explicit request, so it must not drop a - * pending out-of-band one, which is applied when @p next runs. - */ - set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); - __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); -} - /* * Return true if @p is moving due to an internal SCX migration, false * otherwise. @@ -2148,11 +2225,11 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, bool scx_rq_online(struct rq *rq) { /* - * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates - * the online state as seen from the BPF scheduler. cpu_active() test - * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will - * stay set until the current scheduling operation is complete even if - * we aren't locking @rq. + * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE follows the + * core's rq online state. cpu_active() test guarantees that, if this + * function returns %true, %SCX_RQ_ONLINE will stay set until the + * current scheduling operation is complete even if we aren't locking + * @rq. */ return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } @@ -2185,11 +2262,31 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ p->scx.flags &= ~SCX_TASK_IMMED; + /* + * If !scx_rq_online(), we already told the BPF scheduler that the CPU + * is offline and are just running the hotplug path. Don't bother the + * BPF scheduler. + */ + if (!scx_rq_online(rq)) + goto local; + + if (scx_bypassing(sch, cpu_of(rq))) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + goto bypass; + } + /* * A task reenqueued too many times without running means the scheduler * keeps re-deciding a placement it can't honor, e.g. re-inserting to a * cid it lacks caps on. Eject the owning scheduler and strand the task * to be picked up during sched exit. + * + * Bypass guarantees that every runnable task keeps running, and the + * disable relies on that to take sleeping locks. The kernel places the + * tasks of a bypassing scheduler and those on an offline rq, and + * counting their reenqueues could strand a task under bypass and + * deadlock the disable on a lock the task holds, so both tests stay + * above the count. */ if (enq_flags & SCX_ENQ_REENQ) { if (++p->scx.reenq_cnt > 1) @@ -2203,19 +2300,6 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, } } - /* - * If !scx_rq_online(), we already told the BPF scheduler that the CPU - * is offline and are just running the hotplug path. Don't bother the - * BPF scheduler. - */ - if (!scx_rq_online(rq)) - goto local; - - if (scx_bypassing(sch, cpu_of(rq))) { - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - goto bypass; - } - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; @@ -2294,7 +2378,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, dsq = find_global_dsq(sch, task_cpu(p)); goto enqueue; bypass: - dsq = bypass_enq_target_dsq(sch, task_cpu(p)); + dsq = bypass_enq_target_dsq(sch, rq, p, &enq_flags); goto enqueue; enqueue: @@ -2424,7 +2508,13 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) struct scx_sched *sch = scx_task_sched(p); unsigned long opss; - /* dequeue is always temporary, don't reset runnable_at */ + /* + * Don't reset runnable_at. If @p is waiting, the wait continues across + * temporary dequeues, e.g. for a migration or an attribute change. + * scx_reset_task_fields() ends it if @p leaves its sched's control. + * scx_bypass() restarts it when its requeue can change the sched + * running @p. + */ clr_task_runnable(p, false); retry: @@ -2829,7 +2919,6 @@ static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, lockdep_assert_rq_held(rq); WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); - p->scx.holding_cpu = -1; p->scx.flags |= SCX_TASK_REENQ_PROXY; scx_divert_strip_flags(p, &enq_flags); @@ -2863,8 +2952,9 @@ static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, * tell whether we lost to dequeue by testing whether the holding_cpu still * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * - * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid and detached from @dsq. %false if lost to dequeue. + * On return, @dsq is unlocked and @src_rq is locked. Return %true if @p is + * still valid and detached from @dsq, with holding_cpu cleared. Return %false + * if @p was lost to a dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2887,11 +2977,13 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { /* * Keep ->dsq set until we own @src_rq so that a racing dequeue - * can find @dsq and clear holding_cpu. Once ownership is - * confirmed, clear it under @src_rq so that deactivate_task() - * takes the !dsq path instead of retaking @dsq->lock. + * can find @dsq and clear holding_cpu. Once we own @src_rq, the + * handshake is over. Clear ->dsq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. Clear + * holding_cpu with it so that no caller has to. */ p->scx.dsq = NULL; + p->scx.holding_cpu = -1; scx_wait_dispatching(p); return true; } @@ -3127,20 +3219,21 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* the handshake is over, see unlink_dsq_and_switch_rq_lock() */ + p->scx.holding_cpu = -1; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by * abbreviating move_remote_task_to_local_dsq(). */ if (src_rq == dst_rq) { - p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); } else if (unlikely(task_proxy_unsafe_to_move(p))) { fallback = true; scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - p->scx.holding_cpu = -1; fallback = true; scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), p, slice, vtime, @@ -3309,18 +3402,33 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev if (prev->sched_class == &ext_sched_class) { update_curr_scx(rq); + /* + * Under bypass, @prev's slice can't be trusted as a cpu-form + * scheduler can write it directly from BPF. Skipping the keep + * below is not enough as the post-dispatch test in + * scx_dispatch_sched() would keep @prev on the same slice. + * + * Expire it as the bypass tick would, so that neither test + * keeps @prev on it and @prev rotates through the bypass + * enqueue. A kernel slice is left alone and the keep below + * keeps @prev on it, see %SCX_TASK_KERNEL_SLICE. + * + * TODO: Once the cpu form is deprecated and the kfuncs refuse + * slice raises from a bypassing scheduler, slices can be + * trusted under bypass. This expiry, the bypass tick's zeroing + * and the related slice handling can then be cleaned up. + */ + if (scx_bypassing(scx_task_sched(prev), cpu)) + expire_task_slice(prev); + /* * If @prev is runnable & has slice left, it has priority and * fetching more just increases latency for the fetched tasks. * Tell pick_task_scx() to keep running @prev. If the BPF * scheduler wants to handle this explicitly, it should * implement ->cpu_release(). - * - * See scx_disable_workfn() for the explanation on the bypassing - * test. */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(scx_task_sched(prev), cpu)) { + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -3351,6 +3459,34 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev verdict = SCX_DSP_PREV; goto has_tasks; } + + /* + * The scheduler preempts a rescuee like any other task by + * zeroing its slice. The grant survives the preemption. The + * rescuee gets the unserved remainder back and runs again + * behind whatever the scheduler wanted to run first. + * + * Two sites restore the remainder. When another task won the + * pick, put_prev_task_scx() restores it and queues the rescuee + * at the tail of the local DSQ. + * + * When nothing else is runnable, the cpu must not go idle with + * the rescuee queued. The keep above takes a rescuee whose + * scheduler holds %SCX_CAP_BASE here and doesn't take the last + * task through %SCX_OPS_ENQ_LAST. + * + * The pick then ends its zeroed slice, and the rescue with it, + * and refills a default slice. The task continues as an + * ordinary one, which is fine as its scheduler holds BASE here. + * + * For any other rescuee, a reinsert from put_prev_task_scx() + * would come after the pick settled on idle. Keep it running on + * the remainder here. + */ + if (unlikely(prev == scx_rescuee(rq)) && scx_rescue_keep(rq, prev)) { + verdict = SCX_DSP_PREV; + goto has_tasks; + } } rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; @@ -3365,8 +3501,12 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * * - A non-IMMED HEAD task can get queued in front of an IMMED task * between the IMMED queueing and the subsequent scheduling event. + * + * - When dispatch keeps @prev, even a lone IMMED task queued during the + * dispatch can't run right away. */ - if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) + if (unlikely((rq->scx.local_dsq.nr > 1 || verdict == SCX_DSP_PREV) && + rq->scx.nr_immed)) scx_schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; @@ -3388,6 +3528,8 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) p->scx.flags |= SCX_TASK_RUN_TRACKED; } +static void kick_sync_wait_bal_cb(struct rq *rq); + static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) { bool first = type == SNT_PICK; @@ -3403,6 +3545,20 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t (type == SNT_PICK || type == SNT_REPICK)) rq->scx.flags |= SCX_RQ_PROXY_PENDING; + /* + * Defer the kick_sync wait to a balance callback which can drop rq lock + * and enable IRQs. Waiting directly in the pick path would deadlock + * against CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + * + * Queue it only once the pick is final, so that the rq lock can't be + * released with the callback queued (e.g., by another dispatch during + * a core-wide selection). If no SCX task gets picked, there's nothing + * to wait for and kick_sync_pending stays set for the next SCX pick. + */ + if (unlikely(rq->scx.kick_sync_pending) && (type == SNT_PICK || type == SNT_REPICK)) + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); + if (type == SNT_REPICK) return; @@ -3547,6 +3703,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, (rq->scx.flags & SCX_RQ_PROXY_PENDING); bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + bool going_idle = !proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class); rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; @@ -3555,17 +3713,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, update_curr_scx(rq); - /* - * If the slice is consumed, protection ends with it. A rescuee - * preempted beforehand keeps going, see scx_rescue_keep(). - */ - if (!p->scx.slice) { - if (unlikely(p == scx_rescuee(rq))) - rescue_keep = scx_rescue_keep(rq, p); - if (!rescue_keep) - scx_task_slice_ended(rq, p); - } - /* * Preserve the running session when proxy execution refreshes the same * donor around an execution-context switch on this rq. @@ -3578,6 +3725,22 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } + /* + * If the slice is consumed, protection ends with it. + * + * A rescuee keeps going when a preemption or ops.stopping() zeroed its + * slice. Here another task won the pick. scx_rescue_keep() restores the + * unserved remainder and the reinsert below queues the rescuee at the + * tail of the local DSQ. When nothing else is runnable, dispatch_one() + * keeps the rescuee running on the remainder instead. + */ + if (!p->scx.slice) { + if (unlikely(p == scx_rescuee(rq))) + rescue_keep = scx_rescue_keep(rq, p); + if (!rescue_keep) + scx_task_slice_ended(rq, p); + } + if (p->scx.flags & SCX_TASK_QUEUED) { if (proxy_block) goto switch_class; @@ -3603,11 +3766,12 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * the CPU isn't busy, proxy resolution is only retrying through * idle, so keep the donor where it can be picked again. * - * An open rescue must keep @p on the local DSQ even if the - * scheduler zeroed the slice in ops.stopping() above. + * A task with a kernel slice stays on the local DSQ even while + * @sch bypasses. Under bypass, rescue runs @p when the host + * lacks %SCX_CAP_BASE, see bypass_enq_target_dsq(). */ - if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && - !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.slice && (!scx_bypassing(sch, cpu_of(rq)) || + scx_task_has_kernel_slice(p))) { if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); @@ -3615,11 +3779,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, u64 enq_flags = 0; /* - * Keep a preempted rescue going. If preempted - * by another SCX task, append to the local DSQ, - * see scx_rescue_keep(). + * The enqueue caps gate the scheduler's + * placement. The kernel placed a kernel-slice + * task here itself, so reinsert it without a + * cap check, at the head like any preempted + * task. When scx_rescue_keep() restored a slice + * the scheduler zeroed, the scheduler wanted + * something else to run first, so append. */ - if (unlikely(p == scx_rescuee(rq))) { + if (unlikely(scx_task_has_kernel_slice(p))) { enq_flags |= SCX_ENQ_IGNORE_CAPS; if (!rescue_keep) enq_flags |= SCX_ENQ_HEAD; @@ -3647,25 +3815,28 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (!proxy_put && next && - sched_class_above(&ext_sched_class, next->sched_class) && - scx_task_can_stay_on_cpu(rq, p)) { + if (going_idle && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); - - /* - * A task put on the local DSQ runs, as anywhere else. - * Here the insert can't reschedule on its own: the pick - * has settled on @next and @p is still curr, so - * resched_curr() would flag @p and __schedule() clears - * that right after the pick. Flag @next instead. - */ - if (p->scx.dsq == &rq->scx.local_dsq) - set_tsk_need_resched(next); } else { scx_do_enqueue_task(rq, p, 0, -1); } + + /* + * @p may be on the local DSQ now. With %SCX_ENQ_LAST the + * scheduler can put the last runnable task there. The plain + * insert runs when @p cannot stay on this cpu, and rescue can + * still admit it here. + * + * The cpu must not go idle with @p queued on its local DSQ. The + * usual resched doesn't work: the pick has settled on @next and + * @p is still curr, so resched_curr() would set need_resched on + * @p, which __schedule() clears right after the pick. Set it on + * @next instead. + */ + if (going_idle && p->scx.dsq == &rq->scx.local_dsq) + set_tsk_need_resched(next); } switch_class: @@ -3683,17 +3854,22 @@ static void kick_sync_wait_bal_cb(struct rq *rq) s32 cpu; /* - * This callback is queued and normally flushed within @rq's own - * scheduling pass. However, dispatch can drop the rq lock while it sits - * queued, and lock takers in that window (the sched class change paths, - * the scx task iterator) flush pending balance callbacks on release, - * running this one on a foreign CPU whose snapshots are unrelated. The - * kicked CPUs are already on their way to advance the kick_syncs being - * waited on. Don't get in the way. + * This callback is queued once the pick is final and flushed within + * @rq's own scheduling pass. If a lock taker on a foreign CPU ever + * flushes it, the snapshots are unrelated to that CPU. Don't get in the + * way and leave the wait pending for @rq's next pick. */ if (unlikely(cpu_of(rq) != smp_processor_id())) return; + /* + * Clear only when the wait actually runs. The callback can be dropped + * without running (e.g., proxy exec zaps the balance callbacks and + * picks again when find_proxy_task() fails), in which case the next + * pick must queue it again. + */ + rq->scx.kick_sync_pending = false; + ks = __this_cpu_read(scx_kick_syncs); ksyncs = rcu_dereference_sched(ks)->syncs; @@ -3892,25 +4068,12 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * NULL pick from the dl_server, the core continues picking and can * release the rq lock (e.g., prev_balance() or another dispatch). In * that case, run the deferred work from irq_work, which doesn't depend - * on the current rq lock section, and leave the kick sync wait pending - * for the next pick. + * on the current rq lock section. */ if (p != RETRY_TASK && (p || !force_scx)) { /* under core scheduling, see dispatch_core_pick() */ if (!core_sched) maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g., TLB flushes) while we wait for - * them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; schedule_deferred(rq); @@ -4140,6 +4303,7 @@ static void handle_hotplug(struct rq *rq, bool online) s32 cpu_or_cid = cpu; atomic_long_inc(&scx_hotplug_seq); + scx_update_frozen_cpus(cpu, online); /* * scx_root updates are protected by cpus_read_lock() and will stay @@ -4152,11 +4316,6 @@ static void handle_hotplug(struct rq *rq, bool online) if (scx_enabled()) scx_idle_update_selcpu_topology(&sch->ops); - if (online) - scx_online_ecaps(rq); - else - scx_offline_ecaps(rq); - /* * The tables can't be retired while this function is running as the * retirement is inside cpus_read_lock. However, scx_cpu_arg() is @@ -4185,6 +4344,14 @@ static void handle_hotplug(struct rq *rq, bool online) } } + /* + * ecaps are delivered for @cpu only after ops.cpu_online() and before + * ops.cpu_offline(). The ecaps ops may read online_cmask, so + * scx_offline_ecaps() and scx_online_ecaps() run after its update. + */ + if (!online) + scx_offline_ecaps(rq); + if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); else if (!online && SCX_HAS_OP(sch, cpu_offline)) @@ -4194,6 +4361,9 @@ static void handle_hotplug(struct rq *rq, bool online) SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "cpu %d going %s, exiting scheduler", cpu, online ? "online" : "offline"); + + if (online) + scx_online_ecaps(rq); } void scx_rq_activate(struct rq *rq) @@ -4248,24 +4418,17 @@ static bool check_rq_for_timeouts(struct rq *rq) goto out_unlock; list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { - struct scx_sched *sch = scx_task_sched(p); + /* + * @p's owner schedules @p, or its host while the owner is + * bypassing. Test @p against that sched's timeout and blame it. + */ + struct scx_sched *sch = scx_exec_sched(scx_task_sched(p), cpu_of(rq)); unsigned long last_runnable = p->scx.runnable_at; if (unlikely(time_after(jiffies, last_runnable + READ_ONCE(sch->watchdog_timeout)))) { - struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); - /* - * A task can be stuck on a DSQ that a sched other than - * its owner is responsible for draining, e.g. an - * ancestor's bypass DSQ while the owner is bypassing. - * Blame the drainer. The local DSQ is consumed by the - * cpu itself and keeps blame on the owner. - */ - if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL) - sch = dsq->sched; - __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, @@ -4334,7 +4497,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(donor, 0); + expire_task_slice(donor); else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, donor); @@ -4495,6 +4658,34 @@ void scx_enable_task(struct scx_sched *sch, struct task_struct *p) scx_set_task_state(p, SCX_TASK_ENABLED); } +/** + * scx_reset_task_fields - Reset what @p carries from its sched's control + * @rq: @p's rq, locked + * @p: task leaving its sched's control + * + * The vtime, the slice and what rides on it, the reenqueue count and the + * watchdog wait belong to the sched while it controls @p. The control ends when + * @p is taken off the sched or leaves the ext class while staying on the sched. + * Reset them so that @p starts clean wherever it goes. + */ +static void scx_reset_task_fields(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + p->scx.dsq_vtime = 0; + scx_task_slice_ended(rq, p); + scx_set_task_slice(p, 0); + p->scx.reenq_cnt = 0; + + /* + * If @p is runnable but not running, its wait started at runnable_at, + * which ops_dequeue() didn't reset. Leaving the sched's control ends + * the wait. Reset runnable_at the next time @p is enqueued on SCX, so + * that the watchdog doesn't count the time @p spends elsewhere. + */ + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; +} + static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) { struct rq *rq = task_rq(p); @@ -4508,15 +4699,6 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY); - /* - * Reset the SCX-managed fields when @p leaves the BPF scheduler's - * control, after ops.disable() has observed their final values. - */ - p->scx.dsq_vtime = 0; - scx_task_slice_ended(rq, p); - scx_set_task_slice(p, 0); - p->scx.reenq_cnt = 0; - /* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear @@ -4536,6 +4718,11 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) switch (scx_get_task_state(p)) { case SCX_TASK_NONE: + /* + * No op to call, but @p may still carry @sch's state, e.g. a + * task punted by scx_fail_parent() and being rescued. + */ + scx_reset_task_fields(task_rq(p), p); return; case SCX_TASK_INIT: args.cancelled = true; @@ -4544,6 +4731,8 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) break; case SCX_TASK_ENABLED: scx_disable_task(sch, p); + /* after ops.disable() has seen the final values */ + scx_reset_task_fields(task_rq(p), p); break; default: WARN_ON_ONCE(true); @@ -4851,11 +5040,13 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p) * parent's own teardown). There is nothing to disable; calling * scx_disable_task() would WARN on the non-%ENABLED state and trigger a * NONE -> READY validation failure. + * + * Either way, reset @p after ops.disable() has seen the final values. A + * %NONE @p can still be a rescuee under the parent. */ - if (scx_get_task_state(p) == SCX_TASK_NONE) - return; - - scx_disable_task(scx_task_sched(p), p); + if (scx_get_task_state(p) != SCX_TASK_NONE) + scx_disable_task(scx_task_sched(p), p); + scx_reset_task_fields(rq, p); } static void switched_to_scx(struct rq *rq, struct task_struct *p) {} @@ -4938,7 +5129,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in - * scx_do_enqueue_task(), which ejects the task's owning scheduler. + * scx_do_enqueue_task(), which ejects the task's owning scheduler. A bypassing + * scheduler's tasks are placed by the kernel instead and are not counted. */ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, u64 *reenq_flags, u32 *reason) @@ -4948,7 +5140,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || scx_task_has_kernel_slice(p))) return false; *reason = SCX_TASK_REENQ_KFUNC; @@ -5051,7 +5243,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && rq->donor->sched_class == &ext_sched_class && scx_task_reenq_on_cap_revoke(rq, rq->donor)) { - scx_set_task_slice(rq->donor, 0); + expire_task_slice(rq->donor); resched_curr(rq); } @@ -5233,7 +5425,8 @@ static void process_deferred_reenq_users(struct rq *rq) * * A task can be re-rejected repeatedly. Reenqueues are bounded per task by * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning - * scheduler. + * scheduler. A bypassing scheduler's tasks are placed by the kernel instead and + * are not counted. */ static void scx_reenq_reject(struct rq *rq) { @@ -6028,10 +6221,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work) struct scx_dispatch_q *dsq; int cpu, node; - irq_work_sync(&sch->propagate_exit_irq_work); - irq_work_sync(&sch->disable_irq_work); - kthread_destroy_worker(sch->helper); - timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask); free_cpumask_var(sch->stall_cpus); @@ -6049,22 +6238,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work) for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - /* - * $sch would have entered bypass mode before the RCU grace - * period. As that blocks new deferrals, all - * deferred_reenq_local_node's must be off-list by now. - */ - WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - - /* remove the queued ecaps sync so the pcpu can be freed */ - scx_discard_ecaps_to_sync(cpu, pcpu); - - /* - * Bypass blocks new kicks. Flush the kick irq_work so this - * pcpu's to_kick_node is off the list before it is freed. - */ - irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); - WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); @@ -6106,11 +6279,65 @@ static void scx_sched_free_rcu_work(struct work_struct *work) kfree(sch); } +/** + * scx_sched_quiesce_rcu_work - Quiesce a sched and queue its free + * @work: &scx_sched.rcu_work + * + * $sch is freed after two RCU grace periods because of its ecaps sync nodes. + * + * The first grace period runs between scx_kobj_release() and this function. It + * waits for the paths that found $sch before scx_unlink_sched(), such as the + * sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue + * $sch's syncs, and a cpu can take a queued node into a sync batch. The first + * grace period doesn't wait for a batch that started after it began. + * + * Nothing queues $sch's syncs now. Remove the nodes still on the cpus' lists. A + * batch that took a node can still write $sch's pcpu, and a delivery in the + * batch can drop the rq lock before that write. The batch keeps irqs off from + * taking the node to its last pcpu write, so the second grace period waits for + * it before scx_sched_free_rcu_work() frees $sch. + */ +static void scx_sched_quiesce_rcu_work(struct work_struct *work) +{ + struct rcu_work *rcu_work = to_rcu_work(work); + struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); + s32 cpu; + + irq_work_sync(&sch->propagate_exit_irq_work); + irq_work_sync(&sch->disable_irq_work); + kthread_destroy_worker(sch->helper); + timer_shutdown_sync(&sch->bypass_lb_timer); + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * $sch would have entered bypass mode before the first RCU + * grace period. As that blocks new deferrals, all + * deferred_reenq_local_node's must be off-list by now. + */ + WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + + /* remove the queued ecaps sync so the pcpu can be freed */ + scx_discard_ecaps_to_sync(cpu, pcpu); + + /* + * Bypass blocks new kicks. Flush the kick irq_work so this + * pcpu's to_kick_node is off the list before it is freed. + */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); + WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); + } + + INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + queue_rcu_work(system_dfl_wq, &sch->rcu_work); +} + static void scx_kobj_release(struct kobject *kobj) { struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + INIT_RCU_WORK(&sch->rcu_work, scx_sched_quiesce_rcu_work); queue_rcu_work(system_dfl_wq, &sch->rcu_work); } @@ -6414,6 +6641,110 @@ bool scx_hardlockup(int cpu) return true; } +/** + * bypass_lb_reenq_home - Reenqueue a task at its home from the balancer + * @rq: @p's home rq, locked + * @p: task off any DSQ, taken from the bypass DSQ of @dsq_cpu + * @dsq_cpu: cpu whose bypass DSQ gave up @p + * + * @p came off the bypass DSQ of @dsq_cpu with no donee able to take it. A drain + * sends a migration-disabled @p here unless the donee picked for it is its + * home, as every other cpu refuses it. The enqueue sends @p to rescue when the + * host lacks %SCX_CAP_BASE at home and rescue is enabled, and to the home's + * bypass DSQ otherwise. When @p must run at home, migration-disabled or with a + * migration pending, and the host lacks BASE there, it is admitted to the + * home's local DSQ in either case, see bypass_enq_target_dsq(). No DSQ lock may + * be held, as the enqueue takes one. + */ +static void bypass_lb_reenq_home(struct rq *rq, struct task_struct *p, s32 dsq_cpu) +{ + lockdep_assert_rq_held(rq); + + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; + p->scx.reenq_reason_caps = SCX_CAP_BASE; + p->scx.reenq_reason_cid = __scx_cpu_to_cid(dsq_cpu); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); +} + +/** + * bypass_lb_can_move - Test whether @p can move to the bypass DSQ of @donee + * @sch: host sched whose bypass DSQs are balanced + * @p: task on a bypass DSQ of @sch, homed anywhere + * @donee: cpu whose bypass DSQ would receive @p, -1 for none + * + * @p's rq lock pins @p's affinity. Without it the answer is a hint. A task + * whose donee is its own cpu can always run there, and + * task_can_run_on_remote_rq() warns when asked about a task's own cpu. + */ +static bool bypass_lb_can_move(struct scx_sched *sch, struct task_struct *p, s32 donee) +{ + return donee >= 0 && (donee == task_cpu(p) || + task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)); +} + +/** + * bypass_lb_move - Move one task off a bypass DSQ for the balancer + * @sch: host sched whose bypass DSQs are balanced + * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked + * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked + * @p: task on @donor_dsq, homed anywhere + * @donee: cpu whose bypass DSQ receives @p, -1 for none + * + * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The + * caller tests bypass_lb_can_move() without @p's rq lock for a task homed + * elsewhere, and not at all for a drain, so the test is repeated here under + * @p's rq lock. A task that fails it is not put back, as the host may no longer + * consume @donor_dsq. bypass_lb_reenq_home() reenqueues it at its home instead. + * + * The insert and the reenqueue run under @p's rq lock. A task homed on + * @donor_rq is dequeued in place. unlink_dsq_and_switch_rq_lock() takes one + * homed elsewhere off and switches the rq lock from @donor_rq to its own. The + * switch can lose to a dequeue, and @p then belongs to the dequeuer. + * + * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is + * unlocked in between, and @donor_rq too while the lock is switched, so the + * caller's walk must find its place in @donor_dsq again after the call. + * + * Return the cpu that received @p, or -1 if @p was lost to a dequeue. + */ +static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, + struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) +{ + struct rq *rq = task_rq(p); + s32 ret; + + lockdep_assert_rq_held(donor_rq); + lockdep_assert_held(&donor_dsq->lock); + + if (rq == donor_rq) { + scx_wait_dispatching(p); + dispatch_dequeue_locked(p, donor_dsq); + raw_spin_unlock(&donor_dsq->lock); + } else if (!unlink_dsq_and_switch_rq_lock(p, donor_dsq, donor_rq, rq)) { + /* lost to a dequeue, @p is the dequeuer's now */ + ret = -1; + goto out; + } + + if (bypass_lb_can_move(sch, p, donee)) { + scx_dispatch_enqueue(sch, rq, scx_bypass_dsq(sch, donee), p, 0, 0, 0); + ret = donee; + } else { + /* + * No donee, or @p can't run on it. Reenqueue @p at home + * instead, see bypass_lb_reenq_home(). + */ + bypass_lb_reenq_home(rq, p, cpu_of(donor_rq)); + ret = cpu_of(rq); + } +out: + if (rq != donor_rq) + switch_rq_lock(rq, donor_rq); + raw_spin_lock(&donor_dsq->lock); + return ret; +} + static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, struct cpumask *donee_mask, struct cpumask *resched_mask, u32 nr_donor_target, u32 nr_donee_target) @@ -6429,87 +6760,94 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, * All we want to guarantee is reasonable forward progress. No reason to * fine tune. Assuming every task on @donor_dsq runs their full slice, * consider offloading iff the total queued duration is over the - * threshold. + * threshold. A zero target is a drain and empties the DSQ regardless. */ min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; - if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) + if (nr_donor_target && + delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) return 0; raw_spin_rq_lock_irq(donor_rq); raw_spin_lock(&donor_dsq->lock); + /* bound the walk to the tasks queued now, see the seq test below */ + cursor.priv = donor_dsq->seq; list_add(&cursor.node, &donor_dsq->list); resume: n = container_of(&cursor, struct task_struct, scx.dsq_list); n = nldsq_next_task(donor_dsq, n, false); while ((p = n)) { - struct scx_dispatch_q *donee_dsq; - int donee; + s32 donee, cpu; n = nldsq_next_task(donor_dsq, n, false); if (donor_dsq->nr <= nr_donor_target) break; - if (cpumask_empty(donee_mask)) + /* + * Without a donee, a balancing donor has nothing to do. A drain + * still sends its tasks home. + */ + if (cpumask_empty(donee_mask) && nr_donor_target) break; /* - * If an earlier pass placed @p on @donor_dsq from a different - * CPU and the donee hasn't consumed it yet, @p is still on the - * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved - * without its rq locked. Skip. + * A task queued after the walk began, including one the walk + * itself reenqueued, is left for the next pass. */ - if (task_rq(p) != donor_rq) + if (u32_before(cursor.priv, p->scx.dsq_seq)) continue; + /* + * A balancing donor leaves a task that can't move where it is. + * A home reenqueue would push one homed on the donor to the + * back of @donor_dsq on every pass. A drain sends it home. + */ donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) - continue; - - donee_dsq = scx_bypass_dsq(sch, donee); - - /* - * $p's rq is not locked but $p's DSQ lock protects its - * scheduling properties making this test safe. - */ - if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) + donee = -1; + if (nr_donor_target && !bypass_lb_can_move(sch, p, donee)) continue; /* - * Moving $p from one non-local DSQ to another. The source rq - * and DSQ are already locked. Do an abbreviated dequeue and - * then perform enqueue without unlocking $donor_dsq. - * - * We don't want to drop and reacquire the lock on each - * iteration as @donor_dsq can be very long and potentially - * highly contended. Donee DSQs are less likely to be contended. - * The nested locking is safe as only this LB moves tasks - * between bypass DSQs. + * The move drops @donor_dsq's lock, and the rq lock too when + * @p is homed elsewhere. The donor cpu's dispatch and its + * enqueuers can reach @donor_dsq then. Park the cursor behind + * @p and resume from it afterwards. The seq test above keeps a + * task enqueued meanwhile out of the pass. */ - dispatch_dequeue_locked(p, donor_dsq); - scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); + list_move(&cursor.node, &p->scx.dsq_list.node); + cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee); + if (cpu >= 0) { + /* + * @cpu might have been idle and need to be woken up. + * No need to be clever. Reschedule every cpu that + * receives tasks. + */ + cpumask_set_cpu(cpu, resched_mask); - /* - * $donee might have been idle and need to be woken up. No need - * to be clever. Kick every CPU that receives tasks. - */ - cpumask_set_cpu(donee, resched_mask); + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) >= nr_donee_target) + cpumask_clear_cpu(cpu, donee_mask); - if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) - cpumask_clear_cpu(donee, donee_mask); - - nr_balanced++; - if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { - list_move_tail(&cursor.node, &n->scx.dsq_list.node); - raw_spin_unlock(&donor_dsq->lock); - scx_rq_lock_drop(donor_rq); - raw_spin_rq_unlock_irq(donor_rq); - cpu_relax(); - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - goto resume; + /* + * The rq lock is held across the walk, not cycled per + * move. @donor_dsq can be very long and contended, and + * the rq lock is what keeps its consumer and enqueuers + * away while the walk cycles the DSQ lock. The rare + * move of a task homed elsewhere does switch it away, + * see the cursor comment above. The walk can still take + * a while, so let the cpu breathe every so often. + */ + if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) { + raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); + raw_spin_rq_unlock_irq(donor_rq); + cpu_relax(); + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + } } + goto resume; } list_del_init(&cursor.node); @@ -6525,6 +6863,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) const struct cpumask *node_mask = cpumask_of_node(node); struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; + u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; u32 nr_target, nr_donor_target; u32 before_min = U32_MAX, before_max = 0; @@ -6536,13 +6875,15 @@ static void bypass_lb_node(struct scx_sched *sch, int node) u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); nr_tasks += nr; - nr_cpus++; + /* only a cpu whose dispatch consumes its DSQ is capacity */ + if (!scx_missing_caps(sch, cpu, SCX_CAP_BASE)) + nr_cpus++; before_min = min(nr, before_min); before_max = max(nr, before_max); } - if (!nr_cpus) + if (!nr_tasks) return; /* @@ -6550,27 +6891,39 @@ static void bypass_lb_node(struct scx_sched *sch, int node) * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ - nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); + nr_target = nr_cpus ? DIV_ROUND_UP(nr_tasks, nr_cpus) : 0; nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) + /* a donee's DSQ is consumed only if @sch holds %SCX_CAP_BASE */ + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target && + !scx_missing_caps(sch, cpu, SCX_CAP_BASE)) cpumask_set_cpu(cpu, donee_mask); } - /* iterate !donee CPUs and see if they should be offloaded */ + /* + * Offload the !donee cpus. A cpu where the host lost %SCX_CAP_BASE no + * longer consumes its DSQ. Drain it as a donor with a target of zero. A + * zero interval disables the balancing, not the drain. + */ cpumask_clear(resched_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (cpumask_empty(donee_mask)) - break; - if (cpumask_test_cpu(cpu, donee_mask)) + u32 target = nr_donor_target; + + if (unlikely(scx_missing_caps(sch, cpu, SCX_CAP_BASE))) { + /* the mask may predate the loss, keep this cpu out */ + cpumask_clear_cpu(cpu, donee_mask); + target = 0; + } else if (!intv_us || cpumask_empty(donee_mask) || + cpumask_test_cpu(cpu, donee_mask)) { continue; - if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + } + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= target) continue; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, - nr_donor_target, nr_target); + target, nr_target); } for_each_cpu(cpu, resched_mask) @@ -6609,9 +6962,28 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer) for_each_node_with_cpus(node) bypass_lb_node(sch, node); + /* + * A kick during this run can leave the timer pending at now, see + * scx_bypass_lb_kick(). timer_reduce() only brings the expiry forward, + * so that kick is kept. + */ intv_us = READ_ONCE(scx_bypass_lb_intv_us); if (intv_us) - mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); + timer_reduce(timer, jiffies + usecs_to_jiffies(intv_us)); +} + +/** + * scx_bypass_lb_kick - Run the bypass load balancer now + * @sch: sched that lost %SCX_CAP_BASE on a cpu + * + * @sch no longer consumes its bypass DSQ on that cpu. Have the balancer drain + * it without waiting for the next interval, see bypass_lb_node(). A loss during + * a run leaves the timer pending at now, and the run that follows sees it. + */ +void scx_bypass_lb_kick(struct scx_sched *sch) +{ + if (scx_bypass_dsp_enabled(sch)) + mod_timer(&sch->bypass_lb_timer, jiffies); } static bool inc_bypass_depth(struct scx_sched *sch) @@ -6680,11 +7052,12 @@ static void enable_bypass_dsp(struct scx_sched *sch) /* * The LB timer will stop running if bypass dispatch is disabled. Start - * after enabling bypass dispatch. + * after enabling bypass dispatch. A kick from a BASE loss can have set + * the timer to expire now in the meantime, see scx_bypass_lb_kick(). + * timer_reduce() keeps that expiry. */ - if (intv_us && !timer_pending(&host->bypass_lb_timer)) - mod_timer(&host->bypass_lb_timer, - jiffies + usecs_to_jiffies(intv_us)); + if (intv_us) + timer_reduce(&host->bypass_lb_timer, jiffies + usecs_to_jiffies(intv_us)); } /* may be called without holding scx_bypass_lock */ @@ -6757,9 +7130,14 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - ops.dispatch() is ignored. * - * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice - * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue. + * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. + * The bypassed tasks are on the bypass DSQs, under rescue or admitted to a + * local DSQ, out of their reach. + * + * - dispatch_one() expires prev's slice before its keep test as the slice can't + * be trusted. Each tick does the same and rotates the running task to the + * tail of the queue. A kernel slice is left alone, see + * %SCX_TASK_KERNEL_SLICE. * * - pick_next_task() suppresses zero slice warning. * @@ -6836,10 +7214,8 @@ void scx_bypass(struct scx_sched *sch, bool bypass) pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - if (was_bypassing) { + if (was_bypassing) unbypass_renotify_idle(rq, pos, pcpu); - scx_unbypass_replay_ecaps(rq, pos); - } } } @@ -6871,7 +7247,7 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (bypass) scx_task_slice_ended(rq, p); else if (!scx_task_can_stay_on_cpu(rq, p)) - scx_set_task_slice(p, 0); + expire_task_slice(p); } /* @@ -6883,12 +7259,26 @@ void scx_bypass(struct scx_sched *sch, bool bypass) */ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { - if (!scx_is_descendant(scx_task_sched(p), sch)) + struct scx_sched *owner = scx_task_sched(p); + + if (!scx_is_descendant(owner, sch)) continue; /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* nothing */ ; + /* + * The requeue can change the sched running @p, + * and the watchdog counts @p's wait against + * that sched. Restart the wait unless @owner is + * the root, which runs its own tasks bypassing + * or not. + * + * A dispatch in flight writes p->scx.flags + * without @p's rq lock. The dequeue waits it + * out, so set the flag after it. + */ + if (scx_parent(owner)) + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } } @@ -8303,6 +8693,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + scx_init_rq_ops_online(); + /* * Make the scheduler instance visible. Must be inside cpus_read_lock(). * See handle_hotplug(). @@ -9409,6 +9801,20 @@ static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void * { struct scx_sched *sch; + /* track the PM cycle whether or not a root is loaded */ + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + case PM_RESTORE_PREPARE: + scx_sub_pm_begin(); + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + case PM_POST_RESTORE: + scx_sub_pm_end(); + break; + } + guard(rcu)(); sch = rcu_dereference(scx_root); @@ -9476,6 +9882,7 @@ void __init init_sched_ext_class(void) INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); + rq->scx.resched_irq_work = IRQ_WORK_INIT_HARD(resched_irq_workfn); if (cpu_online(cpu)) cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; @@ -10245,10 +10652,12 @@ __bpf_kfunc_start_defs(); * rules above for the ownership model. * * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on - * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is - * always allowed. On the stashed path the slice is packed into an atomic64_t - * with the scheduler id and a flag bit, so a slice too large to fit is clamped - * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. + * @p's cpu, and never on a kernel slice, see %SCX_TASK_KERNEL_SLICE. Shortening + * is honored unless @p is protected, see %SCX_TASK_PROTECTED, which refuses + * every write. A refused write is counted in %SCX_EV_SLICE_DENIED. On the + * stashed path the slice is packed into an atomic64_t with the scheduler id and + * a flag bit, so a slice too large to fit is clamped and counted in + * %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. * * Return %true on success, %false if @p is not on the calling scheduler. */ @@ -10286,9 +10695,13 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return true; } - /* under the rq lock: apply now, extensions gated on baseline access */ + /* + * @p's rq is locked, so apply now. An extension needs baseline access + * and is refused on a kernel slice. + */ if (slice > p->scx.slice && - unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { + unlikely(scx_task_has_kernel_slice(p) || + scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); return true; } @@ -11712,6 +12125,31 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { .filter = scx_kfunc_context_filter, }; +/* + * Kfuncs that tracing programs can't call. A tracing program can run in any + * kernel context, including with an rq lock held. scx_locked_rq() only tracks + * rq locks held across SCX callbacks and returns NULL there. + * + * scx_bpf_cpuperf_set() and scx_bpf_cidperf_set() take the target cpu's rq lock + * when scx_locked_rq() returns NULL. Taking it while the context already holds + * an rq lock, the same one or another cpu's, can deadlock. + * + * Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke + * ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops + * run in the tracing program's context. No SCX op expects to run from an + * arbitrary tracing context. One that calls scx_bpf_cidperf_set() hits the + * deadlock above. + * + * Members must also stay in scx_kfunc_ids_any so that struct_ops and syscall + * programs keep their access. + */ +BTF_KFUNCS_START(scx_kfunc_ids_no_tracing) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_no_tracing) + /* * cpu-form kfuncs that are forbidden from cid-form schedulers * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must @@ -11820,6 +12258,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id); + bool in_no_tracing = btf_id_set8_contains(&scx_kfunc_ids_no_tracing, kfunc_id); u32 moff, flags; /* Not an SCX kfunc - allow. */ @@ -11832,7 +12271,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle || in_cid) ? 0 : -EACCES; + return (!in_no_tracing && (in_any || in_idle || in_cid)) ? 0 : -EACCES; /* * add_subprog_and_kfunc() collects all kfunc calls, including dead code diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 8de9d8fb4b25..1e39d12678c4 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -39,10 +39,18 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, if (scx_consume_global_dsq(sch, rq)) return SCX_DSP_LOCAL; + /* + * A bypass consume puts a task on the cpu being dispatched for, to run + * right away. That is an IMMED insert and needs only + * %SCX_CAP_ENQ_IMMED, the base cap. Dispatch descends into @sch only on + * a cpu where @sch holds the base cap, so @sch holds it here. The same + * holds for the consume at the end of this function. + */ if (scx_bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ if (scx_bypassing(sch, cpu)) { - if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), + SCX_ENQ_IMMED)) return SCX_DSP_LOCAL; return SCX_DSP_NONE; } @@ -63,7 +71,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), + SCX_ENQ_IMMED)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; } @@ -135,7 +144,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), SCX_ENQ_IMMED)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 29e69a83d876..ff0a3d508533 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -878,9 +878,13 @@ struct sched_ext_ops { * @args: argument container, see the struct definition * * The sub-scheduler holds no caps by this point and can no longer - * affect any cid. Its caps went away without a - * sub_child_ecaps_updated() report, so whatever was delegated to it, - * e.g. cpuperf targets, is the parent's to restore here. + * affect any cid. Every cap reported through + * ops.sub_child_ecaps_updated() was also reported lost there before the + * sub-scheduler's tasks came back to this sched. Nothing is reported + * for a sub-scheduler whose enable failed before its tasks were handed + * over. If this sched is being disabled too, it can miss these reports. + * On a cpu whose loss wasn't reported, whatever was delegated, e.g. + * cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -914,6 +918,27 @@ struct sched_ext_ops { * change is in effect on the cpu. Runs in dispatch context with rq lock * held, and can perform all operations allowed in ops.dispatch() * including inserting/moving tasks. + * + * Deliveries start after this sched's tasks are handed to it and before + * it makes its first scheduling decision. The caps granted from + * ops.sub_attach() and any change since are reported then. A cpu that + * isn't online at that point reports them once it comes online. + * Deliveries continue while this sched bypasses. Its tasks are then on + * the bypass DSQs, under rescue or admitted to a local DSQ, where this + * op can't insert or move them. Deliveries stop once this sched is + * being disabled. + * + * Until this sched is being disabled, whenever an op of this sched runs + * under a cid's rq lock, its effective caps on the cid are the @after + * of the last invocation for the cid. + * + * Deliveries for a cid happen only while the root considers the cid's + * cpu online, never before the root's ops.cpu_online() or after its + * ops.cpu_offline() for the cpu. A cpu going offline reports the loss + * of every cap on it, and a cpu coming back reports the regain. A cpu + * that suspend or hibernation takes down reports no loss and keeps its + * caps while down. Only its net change, if any, is reported once it is + * back. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); @@ -940,11 +965,19 @@ struct sched_ext_ops { * @before: the child's effective caps as of the last delivery * @after: the child's effective caps now * - * Invoked right after the child's sub_ecaps_updated(), once a grant or - * revoke is in effect on the cpu. From here on the parent can act on a + * Invoked right after the child's ops.sub_ecaps_updated(), once a + * change is in effect on the cpu. From here on the parent can act on a * cap the child lost, e.g. reset the cpuperf target or schedule on the - * cid. A detaching child's caps go away without a report, see - * sub_detach(). A cpu going offline drops caps without a report. + * cid. A disabling child gets no ops.sub_ecaps_updated(), so only this + * op is invoked for its loss. A parent other than the root loses its + * own caps on the cid too when the cpu goes offline, possibly before + * this op is invoked, and regains them when the cpu comes back, + * possibly after. It can't act on the cid while it holds no caps there. + * + * Deliveries start as for the child's ops.sub_ecaps_updated(), continue + * while either sched bypasses and stop once this sched is being + * disabled. A cpu going offline or coming back is reported as for the + * child's ops.sub_ecaps_updated(). * * Runs in dispatch context with rq lock held, and can perform all * operations allowed in ops.dispatch() including inserting/moving @@ -962,6 +995,7 @@ struct sched_ext_ops { * * @cpu just came online. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs beforehand. + * ops.sub_child_ecaps_updated() runs for @cpu only after this op. */ void (*cpu_online)(s32 cpu); @@ -971,6 +1005,7 @@ struct sched_ext_ops { * * @cpu is going offline. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs afterwards. + * ops.sub_child_ecaps_updated() runs for @cpu only before this op. */ void (*cpu_offline)(s32 cpu); @@ -1309,8 +1344,10 @@ struct scx_event_stats { s64 SCX_EV_SLICE_CLAMPED; /* - * The number of times a slice extension was denied because the - * scheduler lacked baseline cpu access on the task's cpu. + * The number of times a slice write was refused. A raise is refused + * without baseline cpu access on the task's cpu and on a kernel slice. + * Every write is refused on a protected slice, a preemption's zeroing + * included. */ s64 SCX_EV_SLICE_DENIED; @@ -1414,6 +1451,12 @@ struct scx_sched; enum scx_sched_pcpu_flags { SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, + /* ecaps deliveries are enabled, see ecaps_enable() */ + SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1, + /* a report is pending, see ecaps_report() */ + SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2, + /* a cpu going down waits for this sync, see scx_offline_ecaps() */ + SCX_SCHED_PCPU_ECAPS_OFFLINE = 1LLU << 3, }; /* dispatch buf */ @@ -1475,10 +1518,15 @@ struct scx_sched_pcpu { /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; /* - * ecaps as of the last sub_ecaps_updated() and - * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). + * The sched was granted a cap on this cpu at some point. ecaps_report() + * reschedules only these cpus, as rescheduling every cpu on each + * sub-sched enable and disable would disrupt a large machine split many + * ways. Exactly which cpus owe a delivery is hard to track, as + * deliveries are asynchronous and reading caps[] and ecaps doesn't + * tell. A cpu granted a cap at some point is a simple superset. Written + * under pshard->lock. */ - u64 reported_ecaps; + bool had_caps; /* * Decaying rescue runtime consumed on this cpu, see @@ -1751,6 +1799,11 @@ struct scx_sched { bool linked; /* on ->children, see scx_link_sched() */ bool sub_attached; + /* ecaps_enable() has run, under scx_enable_mutex */ + bool ecaps_enabled; + + /* marked cpus whose sync is pending, see ecaps_report() */ + atomic_t nr_ecaps_reports; #endif /* CONFIG_EXT_SUB_SCHED */ /* @@ -2107,12 +2160,13 @@ static const char *scx_enable_state_str[] = { * 4. Lock src_rq (where the task currently lives) * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the * race (dequeue clears holding_cpu to -1 when it takes the task), in - * this case migration is aborted - * 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly - * into dst_rq's local DSQ (no lock swap needed) + * this case migration is aborted. Otherwise the claim is confirmed and + * holding_cpu is cleared here, see unlink_dsq_and_switch_rq_lock() + * 6. If src_rq == dst_rq: enqueue directly into dst_rq's local DSQ (no lock + * swap needed) * 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq - * (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu, - * leave the task on src_rq, and enqueue it on the fallback DSQ. + * (CPU affinity, migration_disabled, etc.). If not, leave the task on + * src_rq, and enqueue it on the fallback DSQ. * 8. Otherwise (i.e. if the task can be moved to dst_rq), call * move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq, * and performs the deactivate/activate migration cycle @@ -2260,6 +2314,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +void scx_resched_curr_deferred(struct rq *rq); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); @@ -2277,6 +2332,7 @@ void scx_cgroup_unlock(void); s32 scx_alloc_kern_arena_objs(struct scx_sched *sch); void scx_disable_bypass_dsp(struct scx_sched *sch); void scx_bypass(struct scx_sched *sch, bool bypass); +void scx_bypass_lb_kick(struct scx_sched *sch); s32 scx_link_sched(struct scx_sched *sch); void scx_unlink_sched(struct scx_sched *sch); void scx_disable_dump(struct scx_sched *sch); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index eb1270f7723c..b5d92d89629b 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -36,12 +36,39 @@ static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ /* latched at root enable before any rescue runs */ -static s32 scx_rescue_bw_1024; +s32 scx_rescue_bw_1024; /* 0 if disabled, see scx_rescue_enabled() */ static s64 scx_rescue_quantum_ns; static s64 scx_rescue_sat_delta_ns; static unsigned long scx_rescue_decay_halflife; static unsigned long scx_rescue_overload_after; +/* + * ecaps_report() waits here for its marked syncs. A cpu going down keeps + * picking tasks after hotplug parks its irq_work thread. Its picks can process + * a marked sync, e.g. a core scheduling sibling's. A regular irq_work queued + * there is left behind on PREEMPT_RT, so the wakeup is hard irq work. A hard + * irq can't take a wait queue's lock, which sleeps on PREEMPT_RT. + * scx_enable_mutex serializes the reports, so the single waiter sleeps on an + * rcuwait. + */ +static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait); + +/* a cpu going down waits here for its marked syncs, see scx_offline_ecaps() */ +static DECLARE_WAIT_QUEUE_HEAD(ecaps_offline_waitq); + +/* + * The cpus that suspend or hibernation took down and that haven't come back. + * Their losses are not delivered, see scx_offline_ecaps(). Hotplug updates the + * set under cpus_write_lock(), and scx_sub_pm_end() clears it under + * cpus_read_lock(). A cpu that fails to come back stays in the set until + * regular hotplug brings it back or the PM cycle ends. + */ +static struct cpumask scx_frozen_cpus; +static DECLARE_WAIT_QUEUE_HEAD(scx_frozen_cpus_waitq); + +/* between the PREPARE and POST notifications of a PM cycle */ +static bool scx_pm_in_progress; + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -401,9 +428,7 @@ void scx_init_root_caps(struct scx_sched *sch) /* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */ static s64 scx_rescue_slice_remaining(struct rq *rq) { - s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap; - - return max(rq->scx.rescue.slice - served, 0); + return max(rq->scx.rescue.slice - rq->scx.rescue.served, 0); } /* @@ -438,6 +463,14 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) lockdep_assert_rq_held(rq); + /* + * Count @delta_exec as served before the clamp below, which bounds only + * the budget charge. update_curr_scx() took the same delta off the + * slice, so a slice set to the unserved remainder, as escalation does, + * reaches zero when the remainder does, see scx_rescue_keep(). + */ + rq->scx.rescue.served += delta_exec; + /* * A rescue slice is bounded by one quantum and tick-driven expiry can * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs. @@ -446,9 +479,13 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; - /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, - cpu_of(rq)); + /* + * The per-cpu usage average feeds the overload victim pick. Charge the + * sched running the rescuee, which is the host while the rescuee's + * owner is bypassing. + */ + pcpu = per_cpu_ptr(scx_exec_sched(scx_task_sched(rq->scx.rescue.curr), + cpu_of(rq))->pcpu, cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -474,19 +511,20 @@ void scx_rescue_end(struct rq *rq) } /** - * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee + * scx_rescue_keep - Keep the rescue going past a zeroed slice * @rq: rq @p is running on - * @p: task under rescue whose slice is exhausted + * @p: task under rescue * - * Called from put_prev_task_scx() to decide what an exhausted slice means for - * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted + * Decide what an exhausted slice means for the rescuee. dispatch_one() calls + * this when nothing else is runnable and put_prev_task_scx() when another task + * won the pick. scx_rescue_charge() ends the rescue the moment the admitted * slice is fully served, so arriving here with the rescue still open means @p - * was preempted. Restore the unserved remainder and return %true - @p stays the - * rescuee and the caller reinserts it at the tail of the local DSQ, behind - * whatever preempted the rescuee. + * was preempted. Restore the unserved remainder and return %true. @p stays the + * rescuee. dispatch_one() keeps it running and put_prev_task_scx() reinserts it + * at the tail of the local DSQ, behind whatever preempted the rescuee. * - * Return %false to end the rescue instead - the slice is already fully served, - * @p is leaving the rq or bypass is dismantling rescues. + * Return %false to end the rescue instead - the slice is already fully served + * or @p is leaving the rq. */ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { @@ -494,11 +532,12 @@ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) lockdep_assert_rq_held(rq); - if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || - scx_bypassing(scx_task_sched(p), cpu_of(rq))) + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED)) return false; - scx_set_task_slice(p, remaining); + /* never called on a protected rescuee, so the write can't be refused */ + if (WARN_ON_ONCE(!scx_set_task_slice(p, remaining))) + return false; return true; } @@ -580,10 +619,10 @@ static void scx_rescue_timer_arm(struct rq *rq) * @p: task being admitted, off any DSQ * @slice: CPU time to grant * - * The schedulers keep their normal control over @p and may preempt or reslice - * it. @slice is measured on served CPU time against the snapshot taken here, so - * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep(). - * Prolonged denial escalates into protected execution, see + * The slice is a kernel slice, see %SCX_TASK_KERNEL_SLICE. The schedulers may + * preempt @p or shorten its slice, never extend it. @slice is measured on + * served CPU time, so neither shortens the rescue, see scx_rescue_charge() and + * scx_rescue_keep(). Prolonged denial escalates into protected execution, see * scx_rescue_timerfn(). */ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) @@ -593,8 +632,9 @@ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) rq->scx.rescue.curr = p; rq->scx.rescue.slice = slice; - rq->scx.rescue.exec_snap = p->se.sum_exec_runtime; + rq->scx.rescue.served = 0; scx_set_task_slice(p, slice); + p->scx.flags |= SCX_TASK_KERNEL_SLICE; scx_rescue_timer_arm(rq); } @@ -629,7 +669,8 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue * demand on this cpu persistently exceeds the configured bandwidth. Eject the * sub with the highest recent rescue consumption instead of letting the - * scheduler stall path blame the waiter's owner, who may just be crowded out. + * scheduler stall path blame the sched running the waiter, which may just be + * crowded out. */ static void scx_rescue_check_overload(struct rq *rq) { @@ -850,9 +891,11 @@ void scx_rescue_init(struct rq *rq) * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled * admission. * - * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued - * to and run by its nearest non-bypassing ancestor. If root is bypassing, it - * always holds all caps. + * bypass_enq_target_dsq() also calls this with the host as @sch when the host + * lacks %SCX_CAP_BASE on @rq's cid, if rescue is enabled or @p must run on the + * cpu, migration-disabled or with a migration pending. The bypassed task is + * then admitted or diverted like a capless %SCX_ENQ_RESCUE insert by the host + * itself. */ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) @@ -895,7 +938,7 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ - if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + if (scx_rescue_enabled() && (*enq_flags & SCX_ENQ_RESCUE)) { __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); if (scx_rescue_try_admit(rq, p)) return &rq->scx.local_dsq; @@ -964,7 +1007,7 @@ static void caps_updated_deliver(struct list_head *to_deliver) * so !has_op means the op is absent or the sched isn't * live yet - e.g. caps grant from ops.sub_attach(). * Either way don't consume - leave for - * scx_sub_seed_caps() to deliver once live. + * scx_sub_seed_caps() to deliver once has_op is set. */ scoped_guard (raw_spinlock, &cu->lock) { if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) && @@ -992,8 +1035,8 @@ static void caps_updated_deliver(struct list_head *to_deliver) /* * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant - * taken during its sub_attach(), before has_op was set). Called once @sch is - * enabled. + * taken during its sub_attach(), before has_op was set). Called once has_op is + * set. */ static void scx_sub_seed_caps(struct scx_sched *sch) { @@ -1030,8 +1073,13 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) * @sch: sched to update * @cid: cid to update * - * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in - * dispatch_one(). + * Queue an ecaps update for @sch's @cid and reschedule the cpu so that it syncs + * in dispatch_one(). The cpu isn't rescheduled while it is inactive or the root + * bypasses, and its next scheduling event processes the sync then. A grant or + * revoke can run in the PM syscore phase, where irq work can malfunction, see + * scx_kick_cpu(), and the root bypasses throughout PM. Callers that need the + * sync processed promptly reschedule the cpu themselves, see + * resched_sync_ecaps(). */ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) { @@ -1039,28 +1087,87 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check - * below sees the node off the list and queues it, or the in-flight sync - * sees the caps[] update made before this call. + * Pairs with smp_mb() in __scx_process_sync_ecaps(). The caller has + * written the state a sync reads, caps[] or cpu_active(). Either the + * check below finds the node off the list and queues a sync, or a sync + * in flight is still ahead of its reads and sees the new state. */ smp_mb(); /* @cid's pshard->lock excludes concurrent queueing attempts */ if (llist_on_list(&pcpu->ecaps_to_sync_node)) return; - if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) - scx_kick_cpu(sch->ancestors[0], cpu, 0); + + /* + * A cpu going down processes its syncs in its own picks, see + * scx_offline_ecaps(), and a resched for every change on the cid would + * only keep preempting its hotplug thread. The irq work needs an online + * cpu. An active cpu stays online here, as @cid's pshard->lock keeps + * preemption off, which holds off the stop_machine() that takes a cpu + * offline. + */ + if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync) && + cpu_active(cpu) && !scx_bypassing(sch->ancestors[0], smp_processor_id())) + scx_resched_curr_deferred(cpu_rq(cpu)); } -/* discard @rq's queued ecaps syncs */ -static void discard_queued_syncs(struct rq *rq) +/* the pshard holding @sch's caps on @cid */ +static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid) { - struct llist_node *pos, *tmp; + return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; +} +static void ecaps_report_irq_workfn(struct irq_work *irq_work) +{ + rcuwait_wake_up(&ecaps_report_wait); +} + +static struct irq_work ecaps_report_irq_work = IRQ_WORK_INIT_HARD(ecaps_report_irq_workfn); + +/* + * Complete @pcpu's report mark. The waiter in ecaps_report() can't be woken + * under the rq lock, so the last completion wakes it through irq_work. + */ +static void ecaps_report_done(struct scx_sched_pcpu *pcpu) +{ + pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_REPORT; + if (atomic_dec_and_test(&pcpu->sch->nr_ecaps_reports)) + irq_work_queue(&ecaps_report_irq_work); +} + +static void ecaps_offline_irq_workfn(struct irq_work *irq_work) +{ + wake_up(&ecaps_offline_waitq); +} + +static DEFINE_IRQ_WORK(ecaps_offline_irq_work, ecaps_offline_irq_workfn); + +/* + * @pcpu's sync marked by scx_offline_ecaps() has been processed or discarded. + * The waiter can't be woken under the rq lock, so the last completion wakes it + * through irq_work. The marks are all done while the cpu's hotplug thread waits + * in scx_offline_ecaps(), before hotplug parks any irq_work thread, so a + * regular irq_work is enough. + */ +static void ecaps_offline_done(struct rq *rq, struct scx_sched_pcpu *pcpu) +{ lockdep_assert_rq_held(rq); - llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) - init_llist_node(pos); + pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_OFFLINE; + WRITE_ONCE(rq->scx.nr_ecaps_offline, rq->scx.nr_ecaps_offline - 1); + if (!rq->scx.nr_ecaps_offline) + irq_work_queue(&ecaps_offline_irq_work); +} + +/* + * Test whether the hotplug operation in progress is suspend or hibernation + * taking a cpu down or bringing it back. A kexec jump also sets + * cpuhp_tasks_frozen but sends no PM notification, so outside a PM cycle its + * cpu downs deliver their losses as in regular hotplug. + */ +static bool scx_pm_frozen(void) +{ + return cpuhp_tasks_frozen && READ_ONCE(scx_pm_in_progress); } /** @@ -1072,6 +1179,11 @@ static void discard_queued_syncs(struct rq *rq) * transposed copy owned by the cid's cpu and written only here under @rq's * lock. * + * A sched's ecaps change only with a delivery, so that whenever an op of the + * sched runs under @rq's lock, its ecaps are what its deliveries reported. A + * sync that can't deliver leaves them alone, and a later one that can reports + * the net change. + * * A sched that newly gains baseline access here is owed an update_idle() so it * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. @@ -1079,35 +1191,23 @@ static void discard_queued_syncs(struct rq *rq) void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); - s32 cid, shard; + s32 cid; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; - /* - * ecaps are zeroed while the cpu is inactive and must stay zero. - * Discard queued syncs instead of processing them - the - * scx_online_ecaps() reseed re-syncs every sched on activation. - * cpu_active() clears before the offline zeroing and sets before the - * reseed is queued, so this test can neither miss a racing sync nor - * eat the reseed. - */ - if (unlikely(!cpu_active(cpu))) { - discard_queued_syncs(rq); - return; - } - - /* @cid is valid here: the cpu is active with queued syncs */ + /* @cid stays valid while a root is enabled */ cid = __scx_cpu_to_cid(cpu); - shard = rcu_dereference_all(scx_cid_to_shard)[cid]; batch = llist_del_all(&rq->scx.ecaps_to_sync); llist_for_each_safe(pos, tmp, batch) { struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); - struct scx_pshard *ps = pcpu->sch->pshard[shard]; + struct scx_pshard *ps = cid_pshard(pcpu->sch, cid); + bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT; + bool offline = pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE; u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -1115,14 +1215,47 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); + /* + * Before the sched's has_op is set, only the parent's op would + * run, and the change would never reach the sched. + * ecaps_enable() enables deliveries on each cpu once has_op is + * set. Deliveries also need the cpu online to the root, see + * handle_hotplug(). + * + * An inactive cpu that can deliver is going down and reads as + * capless. A delivery below can drop the rq lock and the cpu's + * active state can change between syncs, so test it per sync. + * Suspend and hibernation don't deliver the loss of a cpu they + * take down, see scx_offline_ecaps(). + * + * A sync that can't deliver leaves the ecaps alone. It may + * still carry a mark that is cleared below. Don't skip the rest + * of the loop for it. + */ old = READ_ONCE(pcpu->ecaps); - ecaps = calc_effective_caps(ps, cid); + if (!(pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) || + !(rq->scx.flags & SCX_RQ_OPS_ONLINE)) + ecaps = old; + else if (cpu_active(cpu)) + ecaps = calc_effective_caps(ps, cid); + else if (scx_pm_frozen()) + ecaps = old; + else + ecaps = 0; WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; gained = ecaps & ~old; lost_all |= lost; + /* + * The bypass DSQ on this cpu is no longer consumed, so drain + * it. A bypassing sched's own bypass DSQs are empty, as its + * tasks sit on the host's. + */ + if ((lost & SCX_CAP_BASE) && !scx_bypassing(pcpu->sch, cpu)) + scx_bypass_lb_kick(pcpu->sch); + /* * Tell the sched and its parent that the sched's effective caps * on this cid changed. The invocations are equivalent to the @@ -1133,37 +1266,40 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * scheduling can differ from @rq's cpu, so the context is set * up there. The rq recorded in it keeps the dispatches * targeting @rq. - * - * Bypass propagates down the hierarchy, so a sched that isn't - * bypassing has no bypassing parent. The child's bypass state - * gates both deliveries. Both report the same before value, so - * one reported_ecaps covers them. */ - if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + if (ecaps != old) { struct scx_sched *parent = scx_parent(pcpu->sch); struct scx_dsp_ctx *dspc; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + + /* a disabling sched's ops are not called */ + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && + likely(!READ_ONCE(pcpu->sch->aborting))) { dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, - scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_cpu_arg(cpu), old, ecaps); scx_flush_dispatch_buf(pcpu->sch, rq); } - if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + if (SCX_HAS_OP(parent, sub_child_ecaps_updated) && + likely(!READ_ONCE(parent->aborting))) { dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + old, ecaps); scx_flush_dispatch_buf(parent, rq); } rq->scx.sub_dispatch_prev = NULL; - pcpu->reported_ecaps = ecaps; } + if (report) + ecaps_report_done(pcpu); + if (offline) + ecaps_offline_done(rq, pcpu); + /* * Gaining baseline access owes an update_idle() so the sched * learns the cpu's idle state. Arm the per-rq gate so the next @@ -1186,50 +1322,117 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } -/** - * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification - * @rq: rq of the cpu leaving bypass - * @sch: scheduler that just left bypass on @rq's cpu - * - * scx_process_sync_ecaps() consumes syncs while bypassing without delivering - * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a - * sync when bypass lifts, so without a replay a cid that never changes again - * would never be notified. The attach-time initial grants are the acute case - * as they are consumed during the enable bypass window. Re-queue a sync for - * any undelivered delta so the next dispatch delivers it. - */ -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) +void scx_sub_pm_begin(void) { - s32 cpu = cpu_of(rq); - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - struct scx_pshard *ps; - s32 cid; - - lockdep_assert_rq_held(rq); - - /* root holds every cap and never uses ecaps */ - if (!sch->level) - return; - - if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps) - return; - - cid = __scx_cpu_to_cid(cpu); - ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; - - guard(raw_spinlock)(&ps->lock); - queue_sync_ecaps(sch, cid); + WRITE_ONCE(scx_pm_in_progress, true); } /* - * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync - * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() - * only on a real change since offline. + * Called on every cpu down and up whether or not a root is loaded, so that the + * set stays exact when a root is enabled or disabled during a PM cycle. A cpu + * that failed to come back from suspend or hibernation can come back through + * regular hotplug before the PM cycle ends, e.g. through the SMT control. The + * reseed at its activation reports the net change, so any up removes the cpu + * from the set. + */ +void scx_update_frozen_cpus(s32 cpu, bool online) +{ + if (online) { + if (!cpumask_test_cpu(cpu, &scx_frozen_cpus)) + return; + __cpumask_clear_cpu(cpu, &scx_frozen_cpus); + if (cpumask_empty(&scx_frozen_cpus)) + wake_up_all(&scx_frozen_cpus_waitq); + } else if (scx_pm_frozen()) { + __cpumask_set_cpu(cpu, &scx_frozen_cpus); + } +} + +/* + * The PM cycle is over. A cpu still in the set failed to come back, and its + * loss can't be delivered anymore. That breaks the guarantee that every ecaps + * change reaches the sub-scheds and their parents, so abort every sched. A root + * without sub-scheds is owed no reports and keeps running. + * + * Abort before clearing the set and waking the detaches in scx_sub_disable(), + * so that a parent whose ops.sub_detach() runs with that loss unreported is + * already being disabled, which ops.sub_detach() allows for. No exit cpu is + * recorded, as the cpu running this isn't the one that failed. + */ +void scx_sub_pm_end(void) +{ + /* hotplug updates the set under cpus_write_lock() */ + guard(cpus_read_lock)(); + + if (unlikely(!cpumask_empty(&scx_frozen_cpus)) && scx_has_subs()) { + struct scx_sched *root; + + guard(rcu)(); + root = rcu_dereference(scx_root); + if (root) + __scx_exit(root, SCX_EXIT_ERROR, 0, -1, + "%u cpu(s) failed to come back after suspend or hibernation", + cpumask_weight(&scx_frozen_cpus)); + } + + WRITE_ONCE(scx_pm_in_progress, false); + cpumask_clear(&scx_frozen_cpus); + wake_up_all(&scx_frozen_cpus_waitq); +} + +/* + * Called with cpus_read_lock() held before the root is published. The cpus + * active now are online to the root without ops.cpu_online(), so they can + * deliver ecaps. handle_hotplug() maintains %SCX_RQ_OPS_ONLINE from here on. + */ +void scx_init_rq_ops_online(void) +{ + s32 cpu; + + lockdep_assert_cpus_held(); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (cpu_active(cpu)) + rq->scx.flags |= SCX_RQ_OPS_ONLINE; + else + rq->scx.flags &= ~SCX_RQ_OPS_ONLINE; + } +} + +/* + * Make @rq's cpu dispatch through the ext class to process its ecaps syncs. + * queue_sync_ecaps() reschedules only when the cpu has no sync queued yet, + * never an inactive cpu, and not while the root bypasses, so reschedule here. + * The callers run on the cpu's hotplug thread or in ecaps_report(), never in + * the PM syscore phase. A higher class task can hold the cpu, so also start the + * ext server, which picks the ext class within its period. + */ +static void resched_sync_ecaps(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + update_rq_clock(rq); + dl_server_start(&rq->ext_server); + scx_resched_curr_deferred(rq); +} + +/* + * A cpu came back and the root's ops.cpu_online() has run. Let the cpu deliver + * ecaps again and reseed each sub-sched's ecaps on its cid. The sync recomputes + * effective caps from the pshard and reports any change since the last + * delivery. */ void scx_online_ecaps(struct rq *rq) { struct scx_sched *root, *pos; - s32 cid, shard; + s32 cid; + + guard(rq_lock_irqsave)(rq); + + rq->scx.flags |= SCX_RQ_OPS_ONLINE; /* * Only a live hierarchy can have ecaps to reseed. This also keeps the @@ -1240,11 +1443,8 @@ void scx_online_ecaps(struct rq *rq) if (!scx_enabled()) return; - guard(rq_lock_irqsave)(rq); - root = scx_root_protected(); cid = __scx_cpu_to_cid(cpu_of(rq)); - shard = rcu_dereference_all(scx_cid_to_shard)[cid]; scx_for_each_descendant_pre(pos, root) { struct scx_pshard *ps; @@ -1253,48 +1453,93 @@ void scx_online_ecaps(struct rq *rq) if (!pos->level) continue; - ps = pos->pshard[shard]; + ps = cid_pshard(pos, cid); guard(raw_spinlock)(&ps->lock); queue_sync_ecaps(pos, cid); } + + resched_sync_ecaps(rq); } /* - * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks - * treat the cpu as capless while offline. Pending and late-queued syncs are - * discarded at consumption by scx_process_sync_ecaps() while the cpu is - * inactive. Leave reported_ecaps. Ownership is unchanged, so the - * scx_online_ecaps() reseed reports only a genuine delta. No callback fires - * here. + * A cpu is going down, and the root's ops.cpu_offline() follows. Queue a marked + * sync for each sub-sched to report the loss and wait until each has been + * processed. Then stop the cpu's ecaps deliveries, so that none comes after + * ops.cpu_offline(). + * + * This runs on the cpu's hotplug thread. While it sleeps, the cpu keeps + * scheduling, and a pick through the ext class processes the syncs, see + * resched_sync_ecaps(). Only the marked syncs count, so a flood of changes on + * the cid can't hold up the hotplug. After the wait, every sub-sched's ecaps on + * the cpu are zero, and the later syncs have nothing to deliver. + * + * The loss on a cpu that suspend or hibernation takes down is not delivered, so + * that the scheds don't see a PM cycle as cpus leaving and coming back. The cpu + * keeps the caps last delivered until it comes back, and the reseed then + * reports the net change. */ void scx_offline_ecaps(struct rq *rq) { s32 cpu = cpu_of(rq); struct scx_sched *root, *pos; + u32 nr = 0; + s32 cid; + + /* + * ecaps and the tables need a live root, see scx_online_ecaps(). The + * ext class isn't picked without one, so the syncs wouldn't be + * processed and the wait would never end. scx_enabled() can't change + * during hotplug. + */ + if (scx_enabled() && !scx_pm_frozen()) { + scoped_guard (rq_lock_irqsave, rq) { + root = scx_root_protected(); + cid = __scx_cpu_to_cid(cpu); + + scx_for_each_descendant_pre(pos, root) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + struct scx_pshard *ps; + + /* root holds every cap and never uses ecaps */ + if (!pos->level) + continue; + + /* + * A batch that has yet to reach the sync sees + * the mark. A sync already being processed has + * left the list and is queued again below. + */ + pcpu->flags |= SCX_SCHED_PCPU_ECAPS_OFFLINE; + nr++; + + ps = cid_pshard(pos, cid); + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(pos, cid); + } + + WRITE_ONCE(rq->scx.nr_ecaps_offline, nr); + resched_sync_ecaps(rq); + } + + wait_event(ecaps_offline_waitq, !READ_ONCE(rq->scx.nr_ecaps_offline)); + } guard(rq_lock_irqsave)(rq); - - root = scx_root_protected(); - - scx_for_each_descendant_pre(pos, root) { - /* root holds every cap and never uses ecaps */ - if (!pos->level) - continue; - - WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); - } + rq->scx.flags &= ~SCX_RQ_OPS_ONLINE; } /* - * Clear every cap @sch holds. The pshard caps go first as they are the source a - * pending sync recomputes ecaps from. ecaps are then zeroed directly for the - * cap checks. + * Clear every cap @sch holds in the pshards. The report that follows delivers + * the loss and zeroes the ecaps. The caller holds off cpu hotplug across this + * and the report. */ static void clear_all_caps(struct scx_sched *sch) { - s32 si, cpu; + s32 si; u32 cap_bit; + lockdep_assert_cpus_held(); + /* enable may have failed before scx_alloc_pshards() */ if (!sch->pshard) return; @@ -1306,16 +1551,13 @@ static void clear_all_caps(struct scx_sched *sch) for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) scx_cmask_clear(&ps->caps[cap_bit].cmask); } - - for_each_possible_cpu(cpu) { - guard(rq_lock_irqsave)(cpu_rq(cpu)); - WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); - } } /* - * @pcpu's sched was unhashed before the grace period, so nothing re-queues its - * sync node. Remove the node from @rq's pending list so the pcpu can be freed. + * Remove @pcpu's ecaps sync node from @cpu's pending list. The first grace + * period has passed, so nothing queues the node again. The second grace period + * waits for a sync batch that took the node earlier, see + * scx_sched_quiesce_rcu_work(). */ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) { @@ -1323,43 +1565,34 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) struct llist_node *head = NULL, *tail = NULL; struct llist_node *pos, *tmp; + guard(rq_lock_irqsave)(rq); + + /* ecaps_report() waited for every mark it set */ + WARN_ON_ONCE(pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT); + + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + /* * llist can't unlink a single node. Take all queued nodes, drop @pcpu's * and resplice the rest. Nodes in the taken batch read as on-list * throughout, so queue_sync_ecaps() stays correct. */ - if (llist_on_list(&pcpu->ecaps_to_sync_node)) { - scoped_guard (rq_lock_irqsave, rq) { - llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { - if (pos == &pcpu->ecaps_to_sync_node) { - init_llist_node(pos); - } else { - pos->next = head; - head = pos; - if (!tail) - tail = pos; - } - } - if (head) - llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { + if (pos == &pcpu->ecaps_to_sync_node) { + init_llist_node(pos); + /* a cpu going down may be waiting for the sync */ + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE) + ecaps_offline_done(rq, pcpu); + } else { + pos->next = head; + head = pos; + if (!tail) + tail = pos; } } - - /* - * An in-flight scx_process_sync_ecaps() batch may still hold the node - * privately across dispatch-induced rq unlocks, reading as on-list. - * - * Because a bypassing sched gets no op call, init_llist_node() and all - * @pcpu accesses share one contiguous lock hold, off-list under the rq - * lock means @pcpu won't be accessed again. - */ - while (true) { - scoped_guard (rq_lock_irqsave, rq) { - if (!llist_on_list(&pcpu->ecaps_to_sync_node)) - return; - } - cpu_relax(); - } + if (head) + llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); } /** @@ -1378,9 +1611,11 @@ void scx_discard_stale_ecaps_syncs(void) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); + struct llist_node *pos, *tmp; guard(rq_lock_irqsave)(rq); - discard_queued_syncs(rq); + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) + init_llist_node(pos); } } @@ -1435,8 +1670,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) * * Used when @to must take over @p but failed to initialize it. Bypass keeps * scheduling decisions away from @to but @p can still trigger its task ops, - * which may confuse the BPF side. @to is dying anyway. The exit paths skip - * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). + * which may confuse the BPF side. @to is dying anyway. The exit paths call no + * op on a %NONE task. __scx_disable_and_exit_task() and switched_from_scx() + * only reset its fields, see scx_reset_task_fields(). */ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) { @@ -1666,6 +1902,77 @@ static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} #endif +/* + * Enable @sch's ecaps deliveries on every possible cpu, so that a cpu inactive + * during the attach report delivers from the reseed at its activation. + */ +static void ecaps_enable(struct scx_sched *sch) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + per_cpu_ptr(sch->pcpu, cpu)->flags |= SCX_SCHED_PCPU_ECAPS_ENABLED; + } + sch->ecaps_enabled = true; +} + +/** + * ecaps_report - Report @sch's ecaps and wait + * @sch: sub-scheduler to report + * + * Mark every cpu @sch was ever granted a cap on, deliver its pending change + * through the dispatch path and wait for it. An inactive cpu is skipped. It may + * already be offline, where the mark would never be processed and the wait + * would never end. On attach, the reseed at its activation reports its caps. On + * detach, its loss was reported as it went down. A cpu that suspend or + * hibernation took down is back by then, or the hierarchy is aborting, see + * scx_sub_disable(). + * + * If a marked cpu goes down meanwhile, a regular down waits for the marked sync + * in scx_offline_ecaps() while the cpu is still online, as + * ecaps_report_irq_work would never run on an offline cpu. Suspend and + * hibernation don't wait, but no report runs across their downs. The attach + * report runs while the attaching task waits uninterruptibly, which holds off + * the freezer, and the detach report holds cpus_read_lock(). + * + * Called with scx_enable_mutex held. A sched's attach and detach reports must + * not overlap, as the mark is one bit per pcpu. + */ +static void ecaps_report(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + struct rq *rq = cpu_rq(cpu); + struct scx_pshard *ps; + s32 cid; + + guard(rq_lock_irqsave)(rq); + + if (!cpu_active(cpu)) + continue; + + cid = __scx_cpu_to_cid(cpu); + ps = cid_pshard(sch, cid); + guard(raw_spinlock)(&ps->lock); + + if (!pcpu->had_caps) + continue; + + pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT; + atomic_inc(&sch->nr_ecaps_reports); + queue_sync_ecaps(sch, cid); + resched_sync_ecaps(rq); + } + + rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports), + TASK_UNINTERRUPTIBLE); +} + void scx_sub_disable(struct scx_sched *sch) { struct scx_sched *parent = scx_parent(sch); @@ -1687,6 +1994,50 @@ void scx_sub_disable(struct scx_sched *sch) * disable ops. */ mutex_lock(&scx_enable_mutex); + + /* + * Clear @sch's caps and report the loss before any task goes back to + * the parent. A disabling sched is granted nothing, so the caps stay + * cleared. Hold off cpu hotplug across both so that a cpu the report + * skips has finished going down. Its loss was reported as it went down, + * see scx_offline_ecaps(). + * + * The loss on a cpu that suspend or hibernation took down is not + * delivered. Wait for such cpus to come back so that the report reaches + * them. They go down only after the freezer has frozen the tasks and + * come back before the tasks thaw, so the wait doesn't hold up the + * freezer. If one fails to come back, the wait can last until the PM + * cycle ends, and scx_sub_pm_end() aborts every sched first. + * + * The check and the report run in one cpus_read_lock() section so that + * no cpu goes down for PM between them. scx_sub_pm_end() can still + * clear the set meanwhile, which is fine, as it aborts every sched + * first. The report also needs scx_enable_mutex, see ecaps_report(). A + * cpu coming back takes cpus_write_lock(), so the wait drops + * cpus_read_lock(). It drops scx_enable_mutex too so that other enables + * and disables don't wait behind the PM cycle, and retakes both in + * order. + * + * An enable that failed before its attach report has nothing to report + * and doesn't wait. If it failed before scx_link_sched(), the root's + * disable doesn't wait for it and may have retired the cid tables that + * the report reads. + */ + cpus_read_lock(); + while (sch->ecaps_enabled && !data_race(cpumask_empty(&scx_frozen_cpus))) { + cpus_read_unlock(); + mutex_unlock(&scx_enable_mutex); + /* lockless hint, the loop retests */ + wait_event_idle(scx_frozen_cpus_waitq, + data_race(cpumask_empty(&scx_frozen_cpus))); + mutex_lock(&scx_enable_mutex); + cpus_read_lock(); + } + clear_all_caps(sch); + if (sch->ecaps_enabled) + ecaps_report(sch); + cpus_read_unlock(); + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); @@ -1794,12 +2145,6 @@ void scx_sub_disable(struct scx_sched *sch) scx_unlink_sched(sch); - /* - * The parent restores what it delegated in ops.sub_detach(). @sch must - * hold no caps by then. - */ - clear_all_caps(sch); - mutex_unlock(&scx_enable_mutex); /* @@ -1959,15 +2304,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; - /* - * Bypass before @sch is linked and grants can reach it. The parent's - * delivery advances reported_ecaps, so a sync consumed before @sch's - * ops are registered would reach the parent and never be replayed to - * @sch. While bypassed, the syncs are consumed without a delivery and - * replayed to both at unbypass. - */ - scx_bypass(sch, true); - ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -2013,6 +2349,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; + scx_bypass(sch, true); + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -2154,10 +2492,16 @@ void scx_sub_enable_workfn(struct kthread_work *work) scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); - scx_bypass(sch, false); - - /* @sch is enabled; deliver any caps owed since its sub_attach() */ + /* + * Report the caps granted from ops.sub_attach() before bypass lifts and + * @sch makes its first scheduling decision. The report sleeps until its + * syncs are processed, so it runs outside the fork rwsem. + */ scx_sub_seed_caps(sch); + ecaps_enable(sch); + ecaps_report(sch); + + scx_bypass(sch, false); pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); kobject_uevent(&sch->kobj, KOBJ_ADD); @@ -2403,6 +2747,12 @@ __bpf_kfunc_start_defs(); * child schedulers. The child scheduler runs its dispatch operation to * move tasks from dispatch queues to the local runqueue. * + * Dispatch into a child while the local DSQ is empty, when the cpu needs a + * task. A child holding only %SCX_CAP_ENQ_IMMED can insert only IMMED tasks, + * and the consume of its bypassing descendants' bypass DSQs is one such insert. + * With a task already queued on the local DSQ, such an insert fails and the + * task is reenqueued. + * * Returns: true on success, false if cgroup_id is invalid, not a direct * child, or caller lacks dispatch permission. */ @@ -2587,8 +2937,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, * the cid gains baseline access, so the holder * learns of an already-idle cid. */ - scx_cmask_for_each_cid(cid, changed_cids) + scx_cmask_for_each_cid(cid, changed_cids) { + s32 cpu = __scx_cid_to_cpu(cid); + + per_cpu_ptr(child->pcpu, cpu)->had_caps = true; queue_sync_ecaps(child, cid); + } } } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index b76d750463f0..b6d4a522a5a3 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -14,6 +14,8 @@ #ifdef CONFIG_EXT_SUB_SCHED +extern s32 scx_rescue_bw_1024; + struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); @@ -32,7 +34,10 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); +void scx_sub_pm_begin(void); +void scx_update_frozen_cpus(s32 cpu, bool online); +void scx_sub_pm_end(void); +void scx_init_rq_ops_online(void); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); @@ -48,6 +53,12 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq); void scx_rescue_set_knobs(struct scx_sched *sch); void scx_rescue_init(struct rq *rq); +/* false only when the root disabled rescue, see scx_rescue_set_knobs() */ +static inline bool scx_rescue_enabled(void) +{ + return likely(scx_rescue_bw_1024); +} + /* * cgrp->scx_sched is written by root/sub enable/disable under all of * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the @@ -110,7 +121,10 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} -static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_sub_pm_begin(void) {} +static inline void scx_update_frozen_cpus(s32 cpu, bool online) {} +static inline void scx_sub_pm_end(void) {} +static inline void scx_init_rq_ops_online(void) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} @@ -122,6 +136,7 @@ static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_rescue_flush(struct rq *rq) {} static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline bool scx_rescue_enabled(void) { return false; } static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} static inline void scx_rescue_init(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {} @@ -248,12 +263,19 @@ static inline struct task_struct *scx_rescuee(struct rq *rq) return rq->scx.rescue.curr; } +/* does the kernel own @p's slice? see %SCX_TASK_KERNEL_SLICE */ +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) +{ + return scx_has_subs() && (p->scx.flags & SCX_TASK_KERNEL_SLICE); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) { return false; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 87c5335add37..6c7c5633524a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -781,12 +781,21 @@ struct cfs_rq { /* scx_rq->flags, protected by the rq lock */ enum scx_rq_flags { /* - * A hotplugged CPU starts scheduling before rq_online_scx(). Track - * ops.cpu_on/offline() state so that ops.enqueue/dispatch() are called - * only while the BPF scheduler considers the CPU to be online. + * Set while the core has the rq online, see rq_online_scx() and + * rq_offline_scx(). A hotplugged cpu starts scheduling before + * rq_online_scx(). ops.enqueue() and ops.dispatch() are called for the + * cpu only while it is set, see scx_rq_online() and + * scx_dispatch_sched(). */ SCX_RQ_ONLINE = 1 << 0, SCX_RQ_CAN_STOP_TICK = 1 << 1, + /* + * Set while the root considers the cpu online, from its + * ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered + * only while it is set. %SCX_RQ_ONLINE follows the core's rq state + * instead, which changes ahead of the root's view on both edges. + */ + SCX_RQ_OPS_ONLINE = 1 << 2, SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ @@ -807,7 +816,7 @@ struct scx_rq_rescue { u64 clock; /* last budget accrual timestamp */ struct task_struct *curr; /* task being rescued, one at a time */ s64 slice; /* curr's admitted slice */ - u64 exec_snap; /* sum_exec_runtime at admission */ + s64 served; /* curr's served time */ struct timer_list timer; /* paces admission and escalation */ u64 kill_at; /* last ejection, init before any */ }; @@ -838,9 +847,10 @@ struct scx_rq { struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; + u32 nr_ecaps_offline; /* marked syncs a down waits for */ #endif - cpumask_var_t cpus_to_sync; bool kick_sync_pending; + cpumask_var_t cpus_to_sync; unsigned long kick_sync; struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ @@ -855,6 +865,7 @@ struct scx_rq { #endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; + struct irq_work resched_irq_work; }; #endif /* CONFIG_SCHED_CLASS_EXT */ @@ -2167,7 +2178,8 @@ queue_balance_callback(struct rq *rq, * balance_push() is active, see the comment with * balance_push_callback. */ - if (unlikely(head->next || rq->balance_callback == &balance_push_callback)) + if (unlikely(head->next || rq->balance_callback == head || + rq->balance_callback == &balance_push_callback)) return; head->func = func; diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 4fa82033b99c..362e8bb34499 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -108,6 +108,7 @@ #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED #define HAVE_SCX_TASK_RUN_TRACKED +#define HAVE_SCX_TASK_KERNEL_SLICE #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index 1186daa9cb14..ad8f56f581ad 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -120,24 +120,25 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, - { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_KERNEL_SLICE", 0x100LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x19LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, - { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0xe000000LLU }, { "scx_ent_flags", "SCX_TASK_NONE", 0x0LLU }, - { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x100LLU }, - { "scx_ent_flags", "SCX_TASK_INIT", 0x200LLU }, - { "scx_ent_flags", "SCX_TASK_READY", 0x300LLU }, - { "scx_ent_flags", "SCX_TASK_ENABLED", 0x400LLU }, - { "scx_ent_flags", "SCX_TASK_DEAD", 0x500LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xcLLU }, + { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x2000000LLU }, + { "scx_ent_flags", "SCX_TASK_INIT", 0x4000000LLU }, + { "scx_ent_flags", "SCX_TASK_READY", 0x6000000LLU }, + { "scx_ent_flags", "SCX_TASK_ENABLED", 0x8000000LLU }, + { "scx_ent_flags", "SCX_TASK_DEAD", 0xa000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0x1cLLU }, { "scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x3LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x7000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x70000000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x10000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x20000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x30000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x40000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x50000000LLU }, { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index e8e451be862d..63e3c9080e77 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -2180,35 +2180,16 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) { u64 cgid = args->ops->sub_cgroup_id; - s32 nr_cids = qa.nr_cids; - s32 pos = qa.part.rr_pos; - u64 holder_cgid; - bool reset_pool; - s32 i, cid; - - if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) { - scx_bpf_error("-ERANGE"); - return; - } + s32 i; /* - * The child's caps are gone without a report, see ops.sub_detach(). - * cpuperf targets persist until the next write. Reset the child's excl - * cids, and the pool unless another child holds it: that child's revoke - * report or its own detach resets it then. + * qmap_sub_child_ecaps_updated() has reset the cpuperf targets on the + * loss reports. */ - holder_cgid = qa.part.rr_slots[pos]; - reset_pool = !holder_cgid || holder_cgid == cgid; - for (i = 0; i < MAX_SUB_SCHEDS; i++) { if (qa.sub_sched_ctxs[i].cgroup_id != cgid) continue; - bpf_arena_for(cid, 0, nr_cids) - if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) || - (reset_pool && cmask_test(cid, &qa.rr_cids.mask))) - scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); - qa.sub_sched_ctxs[i].cgroup_id = 0; qa.sub_sched_ctxs[i].weight = 100; cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids);