diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5fe980da545d..f1b9d52f624a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1149,6 +1149,10 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + /* @dsq is being destroyed, see free_dsq_irq_workfn() */ + if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID)) + return; + if (list_empty(&dru->node)) list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); WRITE_ONCE(dru->flags, dru->flags | reenq_flags); @@ -2403,11 +2407,14 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, u64 enq_flags, struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); + struct scx_dispatch_q *dst_dsq; lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); + scx_wait_dispatching(p); + + dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) dsq_insert_head(dst_dsq, p); @@ -2587,8 +2594,12 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); + if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + scx_wait_dispatching(p); + return true; + } + + return false; } static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, @@ -2631,6 +2642,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(src_rq); + scx_wait_dispatching(p); + if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); if (src_rq != dst_rq && @@ -3213,8 +3226,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, /* * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell - * ops.enqueue() that @p is the only one available for this cpu, - * which should trigger an explicit follow-up scheduling event. + * ops.enqueue() that @p is the only one available for this cpu. * This doesn't apply if the baseline access on the CPU is lost. * * Under core scheduling, a pick dispatches only when nothing is @@ -3226,6 +3238,16 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + + /* + * A task put on the local DSQ runs, as anywhere else. + * Here the insert can't reschedule on its own: the pick + * has settled on @next and @p is still curr, so + * resched_curr() would flag @p and __schedule() clears + * that right after the pick. Flag @next instead. + */ + if (p->scx.dsq == &rq->scx.local_dsq) + set_tsk_need_resched(next); } else { scx_do_enqueue_task(rq, p, 0, -1); } @@ -4470,11 +4492,12 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, } /* - * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p - * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p - * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + * The dispatcher runs ops.dequeue() and stores the final ops_state after + * dropping the DSQ lock, so @p can be found on a DSQ while still + * %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on + * @p. */ -void scx_reenq_wait_dispatching(struct task_struct *p) +void scx_wait_dispatching(struct task_struct *p) { if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) wait_ops_state(p, SCX_OPSS_DISPATCHING); @@ -4523,7 +4546,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; - scx_reenq_wait_dispatching(p); + scx_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4647,7 +4670,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ - scx_reenq_wait_dispatching(p); + scx_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); @@ -5171,8 +5194,8 @@ static void exit_dsq(struct scx_dispatch_q *dsq) struct rq *rq = cpu_rq(cpu); /* - * There must have been a RCU grace period since the last - * insertion and @dsq should be off the deferred list by now. + * free_dsq_irq_workfn() cancelled the reenqs before the grace + * period and schedule_dsq_reenq() queues nothing after that. */ if (WARN_ON_ONCE(!list_empty(&dru->node))) { guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); @@ -5196,8 +5219,26 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work) struct llist_node *to_free = llist_del_all(&dsqs_to_free); struct scx_dispatch_q *dsq, *tmp_dsq; - llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) { + s32 cpu; + + /* + * Cancel pending reenqs. Only a run_deferred() that started + * before the grace period can hold one, and it keeps IRQs off, + * so the grace period waits for it. After this sweep, + * schedule_dsq_reenq() sees the invalid id under the same lock + * and queues nothing. + */ + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct rq *rq = cpu_rq(cpu); + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + list_del_init(&pcpu->deferred_reenq_user.node); + } + call_rcu(&dsq->rcu, free_dsq_rcufn); + } } static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); @@ -6068,15 +6109,12 @@ static void enable_bypass_dsp(struct scx_sched *sch) /* * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of - * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is - * called iff @sch is not already bypassed due to an ancestor bypassing, - * we can assume that the parent is not bypassing and thus will be the - * host of the bypass DSQs. - * - * While the situation may change in the future, the following - * guarantees that the nearest non-bypassing ancestor or root has bypass - * dispatch enabled while a descendant is bypassing, which is all that's - * required. + * the nearest non-bypassing ancestor or root. The parent hosts them + * unless it is bypassing too. Then they sit further up and the parent + * consumes only its own bypass DSQs, empty until it leaves bypass and + * becomes the host. Either way the nearest non-bypassing ancestor or + * root has bypass dispatch enabled while a descendant is bypassing, + * which is all that's required. * * scx_bypass_dsp_enabled() test is used to determine whether to enter * the bypass dispatch handling path from both bypassing and hosting @@ -6185,34 +6223,45 @@ void scx_bypass(struct scx_sched *sch, bool bypass) struct scx_sched *pos; unsigned long flags; int cpu; + bool changed; raw_spin_lock_irqsave(&scx_bypass_lock, flags); if (bypass) { - if (!inc_bypass_depth(sch)) - goto unlock; - - enable_bypass_dsp(sch); + changed = inc_bypass_depth(sch); + if (changed) + enable_bypass_dsp(sch); } else { - if (!dec_bypass_depth(sch)) - goto unlock; + changed = dec_bypass_depth(sch); } /* * Bypass state is propagated to all descendants - an scx_sched bypasses - * if itself or any of its ancestors are in bypass mode. + * if itself or any of its ancestors are in bypass mode. A descendant's + * depth is the sum of its own bypasses and its ancestors', so every + * call propagates whether or not @sch changed state. The descendants' + * claims are released below with @sch's, after their tasks have moved. */ raw_spin_lock(&scx_sched_lock); scx_for_each_descendant_pre(pos, sch) { if (pos == sch) continue; - if (bypass) - inc_bypass_depth(pos); - else + if (bypass) { + if (inc_bypass_depth(pos)) + enable_bypass_dsp(pos); + } else { dec_bypass_depth(pos); + } } raw_spin_unlock(&scx_sched_lock); + /* + * A descendant's depth is at least @sch's, so no bypass state changed + * unless @sch's did. + */ + if (!changed) + goto unlock; + /* * No task property is changing. We just need to make sure all currently * queued tasks are re-queued according to the new scx_bypassing() @@ -6291,8 +6340,12 @@ void scx_bypass(struct scx_sched *sch, bool bypass) } /* disarming must come after moving all tasks out of the bypass DSQs */ - if (!bypass) - scx_disable_bypass_dsp(sch); + if (!bypass) { + guard(raw_spinlock)(&scx_sched_lock); + scx_for_each_descendant_pre(pos, sch) + if (!pos->bypass_depth) + scx_disable_bypass_dsp(pos); + } unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index aa9fb6de0ad6..28ed9e327085 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -952,12 +952,43 @@ static bool is_bpf_migration_disabled(const struct task_struct *p) return p->migration_disabled; } +static s32 select_cpu_restricted(struct task_struct *p, s32 prev_cpu, + const struct cpumask *allowed, u64 flags, + bool migration_disabled) +{ + s32 cpu; + + if (migration_disabled) + cpu = task_cpu(p); + else + cpu = cpumask_first(p->cpus_ptr); + + if (!cpumask_test_cpu(cpu, p->cpus_ptr)) + return -EBUSY; + + if (allowed && !cpumask_test_cpu(cpu, allowed)) + return -EBUSY; + + if (flags & SCX_PICK_IDLE_IN_NODE) { + int node = scx_cpu_node_if_enabled(prev_cpu); + + if (scx_cpu_node_if_enabled(cpu) != node) + return -EBUSY; + } + + if (!scx_idle_test_and_clear_cpu(cpu)) + return -EBUSY; + + return cpu; +} + static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, s32 prev_cpu, u64 wake_flags, const struct cpumask *allowed, u64 flags) { unsigned long irq_flags; bool we_locked = false; + bool migration_disabled; s32 cpu; if (!scx_cpu_valid(sch, prev_cpu, NULL)) @@ -994,20 +1025,17 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, /* * This may also be called from ops.enqueue(), so we need to handle - * per-CPU tasks as well. For these tasks, we can skip all idle CPU - * selection optimizations and simply check whether the previously - * used CPU is idle and within the allowed cpumask. + * per-CPU tasks as well. For these tasks, skip general idle CPU + * selection and validate the task's only usable CPU directly. */ - if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { - if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && - scx_idle_test_and_clear_cpu(prev_cpu)) - cpu = prev_cpu; - else - cpu = -EBUSY; - } else { + migration_disabled = is_bpf_migration_disabled(p); + + if (p->nr_cpus_allowed == 1 || migration_disabled) + cpu = select_cpu_restricted(p, prev_cpu, allowed, flags, + migration_disabled); + else cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, allowed ?: p->cpus_ptr, flags); - } if (we_locked) raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 5b37faa532d6..352547aedbb7 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1774,8 +1774,9 @@ enum scx_enq_flags { * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the * %SCX_ENQ_LAST flag set. * - * The BPF scheduler is responsible for triggering a follow-up - * scheduling event. Otherwise, Execution may stall. + * If the task is queued on the local DSQ of the CPU it was running on, + * it continues to run. Otherwise, the CPU goes idle. A scheduler that + * wants a full dispatch cycle on the CPU should kick it. */ SCX_ENQ_LAST = 1LLU << 41, @@ -2122,7 +2123,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); -void scx_reenq_wait_dispatching(struct task_struct *p); +void scx_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 90441206d3b9..212de1cb8502 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -797,7 +797,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; - scx_reenq_wait_dispatching(p); + scx_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))