diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 582d7cd4a983..36c04797436c 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -207,6 +207,7 @@ struct sched_ext_entity { s32 holding_cpu; s32 selected_cpu; s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ + u32 ops_qseq; /* protected by rq lock */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e56c3c95018f..5fe980da545d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2057,8 +2057,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (unlikely(!SCX_HAS_OP(sch, enqueue))) goto global; - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + /* + * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the + * per-task qseq counter where the QSEQ field wraps and skip 0, which is + * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING. + */ + p->scx.ops_qseq = ((p->scx.ops_qseq + 1) & + (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1; + qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT; WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); @@ -2123,8 +2129,8 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p) } /* - * list_add_tail() must be used. scx_bypass() depends on tasks being - * appended to the runnable_list. + * list_add_tail() must be used. scx_bypass() and rq_offline_scx() + * depend on tasks being appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); @@ -2152,6 +2158,13 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; + /* + * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can + * leave custody when inserted into the destination DSQ. + */ + if (sticky_cpu >= 0) + p->scx.sticky_cpu = -1; + /* * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue * returns. Only the core's wakeup path delivers one. The flags stashed @@ -2190,9 +2203,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ dl_server_start(&rq->ext_server); scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); - - if (sticky_cpu >= 0) - p->scx.sticky_cpu = -1; out: rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; @@ -3731,8 +3741,26 @@ static void rq_online_scx(struct rq *rq) static void rq_offline_scx(struct rq *rq) { + struct task_struct *p, *n; + rq->scx.flags &= ~SCX_RQ_ONLINE; + + /* sched domain rebuilds call rq_offline with the CPU staying alive */ + if (cpu_active(cpu_of(rq))) + return; + scx_rescue_flush(rq); + + /* + * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks + * onto the local DSQ so that they run here and balance_push() moves + * them off. + */ + list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { + if (p->scx.dsq == &rq->scx.local_dsq) + continue; + guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK); + } } static bool check_rq_for_timeouts(struct rq *rq) @@ -6947,9 +6975,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, - rq->scx.ops_qseq, rq->scx.kick_sync); + rq->scx.kick_sync); scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 2ff5479334cb..8de9d8fb4b25 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -70,7 +70,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, #endif /* CONFIG_EXT_SUB_SCHED */ } - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + /* + * scx_rq_online() can't be used. Its cpu_active() test goes false + * before CPU hotplug waits for an RCU grace period, and + * rq_offline_scx() moves this CPU's tasks to the local DSQ only after + * the wait. The grace period can depend on those tasks running. + */ + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE)) return SCX_DSP_NONE; dspc->rq = rq; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1df8f583b0ec..5b37faa532d6 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1998,6 +1998,11 @@ enum scx_ops_state { * dequeue/requeue, the dispatcher can tell whether it still has a claim * on the task being dispatched. * + * QSEQ is generated from the per-task p->scx.ops_qseq counter so that + * it doesn't repeat across QUEUED instances of the same task even if + * the task moves between rqs. 0 is never used as a valid QSEQ since + * NONE and DISPATCHING map to this value. + * * As some 32bit archs can't do 64bit store_release/load_acquire, * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on * 32bit machines. The dispatch race window QSEQ protects is very narrow diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0472eaf41c7c..90441206d3b9 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -585,10 +585,6 @@ void scx_rescue_flush(struct rq *rq) lockdep_assert_rq_held(rq); - /* sched domain rebuilds call rq_offline with the CPU staying alive */ - if (cpu_active(cpu_of(rq))) - return; - /* end the current rescue */ if (rq->scx.rescue.curr) scx_task_slice_ended(rq, rq->scx.rescue.curr); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e656c7059bf8..4c25fbe84fb5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -816,7 +816,6 @@ struct scx_rq { #endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ - unsigned long ops_qseq; /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ u64 remote_activate_enq_flags; struct scx_sched *remote_activate_sch; @@ -4222,8 +4221,9 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head); * after which it is enqueued again. * * Typically this must be called while holding task_rq_lock, since most/all - * properties are serialized under those locks. There is currently one - * exception to this rule in sched/ext which only holds rq->lock. + * properties are serialized under those locks. There are currently two + * exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass() + * and rq_offline_scx(). */ /* diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 4e06d0baaeec..af919a56a4f4 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -165,6 +165,7 @@ auto-test-targets := \ create_dsq \ dequeue \ dequeue_iter \ + dequeue_remote \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ diff --git a/tools/testing/selftests/sched_ext/dequeue_remote.bpf.c b/tools/testing/selftests/sched_ext/dequeue_remote.bpf.c new file mode 100644 index 000000000000..983590c6d498 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_remote.bpf.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verify that ops.dequeue() is called when a task leaves the BPF scheduler's + * custody by being moved to the local DSQ of a CPU other than the one whose + * rq it's on (move_remote_task_to_local_dsq()). + * + * ops.enqueue() puts every task into custody and ops.dispatch() moves it to + * the dispatching CPU's local DSQ, so most moves cross CPUs. With + * @test_use_move_to_local, tasks are queued on a user DSQ and consumed with + * scx_bpf_dsq_move_to_local(). Otherwise, they are queued in a BPF queue and + * dispatched with SCX_DSQ_LOCAL_ON. + * + * Copyright (c) 2026 Google LLC. + */ + +#include + +#define SHARED_DSQ 0 +#define MAX_DISPATCH_POPS 8 + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 32768); + __type(value, s32); +} global_queue SEC(".maps"); + +enum task_state { + TASK_NONE = 0, + TASK_ENQUEUED, /* in BPF custody, waiting for ops.dequeue() */ + TASK_DISPATCHED, /* left custody */ +}; + +struct task_ctx { + enum task_state state; + s32 enq_cpu; /* scx_bpf_task_cpu() at ops.enqueue() */ + u64 enqueue_seq; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +/* core_cookie only exists with CONFIG_SCHED_CORE */ +struct task_struct___core_sched { + unsigned long core_cookie; +} __attribute__((preserve_access_index)); + +bool test_use_move_to_local; + +u64 enqueue_cnt, dequeue_cnt, dispatch_dequeue_cnt, change_dequeue_cnt; +u64 remote_dispatch_cnt, remote_running_cnt, missed_dequeue_cnt; +u64 core_sched_exec_dequeue_cnt; + +static struct task_ctx *lookup_task_ctx(struct task_struct *p) +{ + return bpf_task_storage_get(&task_ctx_stor, p, 0, 0); +} + +static bool task_has_core_cookie(struct task_struct *p) +{ + struct task_struct___core_sched *t = (void *)p; + + if (!bpf_core_field_exists(t->core_cookie)) + return false; + return BPF_CORE_READ(t, core_cookie); +} + +s32 BPF_STRUCT_OPS(dequeue_remote_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + /* no direct dispatch, always go through ops.enqueue() */ + return prev_cpu; +} + +void BPF_STRUCT_OPS(dequeue_remote_enqueue, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx; + s32 pid = p->pid; + + tctx = lookup_task_ctx(p); + if (!tctx) { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + return; + } + + /* the previous custody period must have ended with ops.dequeue() */ + if (tctx->state == TASK_ENQUEUED) + scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu", + p->pid, p->comm, tctx->enqueue_seq); + + /* + * Mark @p as enqueued before making it visible to ops.dispatch() on + * other CPUs, which skips queue entries of tasks not in ENQUEUED + * state as stale. + */ + tctx->state = TASK_ENQUEUED; + tctx->enq_cpu = scx_bpf_task_cpu(p); + tctx->enqueue_seq++; + + if (test_use_move_to_local) { + scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags); + } else if (bpf_map_push_elem(&global_queue, &pid, 0)) { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + tctx->state = TASK_DISPATCHED; + tctx->enq_cpu = -1; + goto out; + } + + __sync_fetch_and_add(&enqueue_cnt, 1); +out: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(dequeue_remote_dequeue, struct task_struct *p, u64 deq_flags) +{ + struct task_ctx *tctx; + + __sync_fetch_and_add(&dequeue_cnt, 1); + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + /* + * Only core scheduling can pick a task straight out of custody, and + * only if the task has a core cookie. Otherwise, the custody exit was + * missed when @p was inserted into a local DSQ and got deferred until + * @p was picked. + */ + if ((deq_flags & SCX_DEQ_CORE_SCHED_EXEC) && !task_has_core_cookie(p)) { + __sync_fetch_and_add(&core_sched_exec_dequeue_cnt, 1); + scx_bpf_error("%d (%s): late ops.dequeue() with SCX_DEQ_CORE_SCHED_EXEC (enq_cpu=%d cpu=%d seq=%llu)", + p->pid, p->comm, tctx->enq_cpu, + scx_bpf_task_cpu(p), tctx->enqueue_seq); + } + + /* ops.dequeue() ends the custody period started by ops.enqueue() */ + if (tctx->state != TASK_ENQUEUED) + scx_bpf_error("%d (%s): dequeue outside custody deq_flags=0x%llx state=%d seq=%llu", + p->pid, p->comm, deq_flags, tctx->state, + tctx->enqueue_seq); + + if (deq_flags & SCX_DEQ_SCHED_CHANGE) { + __sync_fetch_and_add(&change_dequeue_cnt, 1); + tctx->state = TASK_NONE; + } else { + __sync_fetch_and_add(&dispatch_dequeue_cnt, 1); + tctx->state = TASK_DISPATCHED; + } +} + +void BPF_STRUCT_OPS(dequeue_remote_dispatch, s32 cpu, struct task_struct *prev) +{ + struct task_ctx *tctx; + struct task_struct *p; + s32 pid; + int i; + + if (test_use_move_to_local) { + scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); + return; + } + + /* pop past stale entries so that they don't leave this CPU idle */ + bpf_for(i, 0, MAX_DISPATCH_POPS) { + if (bpf_map_pop_elem(&global_queue, &pid)) + return; + + p = bpf_task_from_pid(pid); + if (!p) + continue; + + /* + * Entries are stale if @p left custody through a property + * change dequeue or was dispatched from a duplicate entry. + */ + tctx = lookup_task_ctx(p); + if (!tctx || tctx->state != TASK_ENQUEUED) { + bpf_task_release(p); + continue; + } + + if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { + if (scx_bpf_task_cpu(p) != cpu) + __sync_fetch_and_add(&remote_dispatch_cnt, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cpu, + SCX_SLICE_DFL, 0); + } else { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); + } + + bpf_task_release(p); + return; + } + + /* out of pops with entries left, retry instead of idling this CPU */ + if (!bpf_map_peek_elem(&global_queue, &pid)) + scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(dequeue_remote_running, struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + /* tasks can only run from a local DSQ, i.e. after leaving custody */ + if (tctx->state == TASK_ENQUEUED) { + __sync_fetch_and_add(&missed_dequeue_cnt, 1); + scx_bpf_error("%d (%s): running without ops.dequeue() (enq_cpu=%d cpu=%d seq=%llu)", + p->pid, p->comm, tctx->enq_cpu, + scx_bpf_task_cpu(p), tctx->enqueue_seq); + return; + } + + if (tctx->enq_cpu >= 0 && tctx->enq_cpu != scx_bpf_task_cpu(p)) + __sync_fetch_and_add(&remote_running_cnt, 1); + tctx->enq_cpu = -1; +} + +s32 BPF_STRUCT_OPS(dequeue_remote_init_task, struct task_struct *p, + struct scx_init_task_args *args) +{ + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!tctx) + return -ENOMEM; + + /* task storage persists across attachments, start from scratch */ + tctx->state = TASK_NONE; + tctx->enq_cpu = -1; + tctx->enqueue_seq = 0; + + return 0; +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_remote_init) +{ + return scx_bpf_create_dsq(SHARED_DSQ, -1); +} + +void BPF_STRUCT_OPS(dequeue_remote_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops dequeue_remote_ops = { + .select_cpu = (void *)dequeue_remote_select_cpu, + .enqueue = (void *)dequeue_remote_enqueue, + .dequeue = (void *)dequeue_remote_dequeue, + .dispatch = (void *)dequeue_remote_dispatch, + .running = (void *)dequeue_remote_running, + .init_task = (void *)dequeue_remote_init_task, + .init = (void *)dequeue_remote_init, + .exit = (void *)dequeue_remote_exit, + .flags = SCX_OPS_ENQ_LAST, + .name = "dequeue_remote", +}; diff --git a/tools/testing/selftests/sched_ext/dequeue_remote.c b/tools/testing/selftests/sched_ext/dequeue_remote.c new file mode 100644 index 000000000000..f2c1901030de --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_remote.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verify that ops.dequeue() is called for tasks leaving BPF custody through + * an SCX-internal cross-CPU migration (move_remote_task_to_local_dsq()). + * + * Copyright (c) 2026 Google LLC. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "scx_test.h" +#include "dequeue_remote.bpf.skel.h" + +#define MAX_WORKERS 64 +#define RUN_MS 2000 + +static int nr_cpus; + +static long long now_ms(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000; +} + +/* mix of short bursts and sleeps to generate lots of enqueues and wakeups */ +static void worker_fn(int id) +{ + long long end = now_ms() + RUN_MS; + volatile unsigned long sum = 0; + + while (now_ms() < end) { + unsigned long j; + + for (j = 0; j < 20000 + id * 1000; j++) + sum += j; + if (id & 1) + usleep(100); + else + sched_yield(); + } + + exit(0); +} + +static enum scx_test_status run_scenario(struct dequeue_remote *skel, + bool use_move_to_local, + const char *name) +{ + enum scx_test_status ret = SCX_TEST_PASS; + struct bpf_link *link; + pid_t pids[MAX_WORKERS]; + int nr_workers, nr_forked, i; + + nr_workers = 2 * nr_cpus; + if (nr_workers < 4) + nr_workers = 4; + if (nr_workers > MAX_WORKERS) + nr_workers = MAX_WORKERS; + + skel->bss->test_use_move_to_local = use_move_to_local; + skel->bss->enqueue_cnt = 0; + skel->bss->dequeue_cnt = 0; + skel->bss->dispatch_dequeue_cnt = 0; + skel->bss->change_dequeue_cnt = 0; + skel->bss->remote_dispatch_cnt = 0; + skel->bss->remote_running_cnt = 0; + skel->bss->missed_dequeue_cnt = 0; + skel->bss->core_sched_exec_dequeue_cnt = 0; + memset(&skel->data->uei, 0, sizeof(skel->data->uei)); + + link = bpf_map__attach_struct_ops(skel->maps.dequeue_remote_ops); + SCX_FAIL_IF(!link, "Failed to attach struct_ops for %s", name); + + fflush(stdout); + fflush(stderr); + + for (nr_forked = 0; nr_forked < nr_workers; nr_forked++) { + pid_t pid = fork(); + + if (pid < 0) { + SCX_ERR("Failed to fork worker %d", nr_forked); + ret = SCX_TEST_FAIL; + break; + } + if (pid == 0) + worker_fn(nr_forked); + pids[nr_forked] = pid; + } + + /* on failure, kill the remaining workers but still reap them */ + for (i = 0; i < nr_forked; i++) { + int status; + + if (ret != SCX_TEST_PASS) + kill(pids[i], SIGKILL); + + if (waitpid(pids[i], &status, 0) != pids[i]) { + SCX_ERR("Failed to wait for worker %d", i); + ret = SCX_TEST_FAIL; + } else if (ret == SCX_TEST_PASS && status != 0) { + SCX_ERR("Worker %d exited with status %d", i, status); + ret = SCX_TEST_FAIL; + } + } + + bpf_link__destroy(link); + + if (ret != SCX_TEST_PASS) + return ret; + + printf("%s:\n", name); + printf(" workers: %d\n", nr_workers); + printf(" enqueues: %lu\n", (unsigned long)skel->bss->enqueue_cnt); + printf(" dequeues: %lu (dispatch: %lu, property_change: %lu)\n", + (unsigned long)skel->bss->dequeue_cnt, + (unsigned long)skel->bss->dispatch_dequeue_cnt, + (unsigned long)skel->bss->change_dequeue_cnt); + if (!use_move_to_local) + printf(" remote SCX_DSQ_LOCAL_ON dispatches: %lu\n", + (unsigned long)skel->bss->remote_dispatch_cnt); + printf(" ran on a CPU other than the enqueue CPU: %lu\n", + (unsigned long)skel->bss->remote_running_cnt); + printf(" ran without ops.dequeue(): %lu\n", + (unsigned long)skel->bss->missed_dequeue_cnt); + printf(" late SCX_DEQ_CORE_SCHED_EXEC dequeues: %lu\n", + (unsigned long)skel->bss->core_sched_exec_dequeue_cnt); + + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG)) + SCX_ERR("Scheduler exited with kind=%lld: %s", + (long long)skel->data->uei.kind, skel->data->uei.msg); + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + + /* the test is meaningless if no task was moved across CPUs */ + SCX_GT(skel->bss->remote_running_cnt, 0); + SCX_EQ(skel->bss->enqueue_cnt, skel->bss->dequeue_cnt); + + return SCX_TEST_PASS; +} + +static enum scx_test_status setup(void **ctx) +{ + struct dequeue_remote *skel; + cpu_set_t cpus; + + SCX_FAIL_IF(sched_getaffinity(0, sizeof(cpus), &cpus), + "Failed to get CPU affinity"); + nr_cpus = CPU_COUNT(&cpus); + if (nr_cpus < 2) { + fprintf(stderr, "Skipping: requires at least 2 usable CPUs\n"); + return SCX_TEST_SKIP; + } + + skel = SCX_OPS_OPEN(dequeue_remote_ops, dequeue_remote); + SCX_OPS_LOAD(skel, dequeue_remote_ops, dequeue_remote, uei); + + *ctx = skel; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + struct dequeue_remote *skel = ctx; + enum scx_test_status status; + + status = run_scenario(skel, false, + "BPF queue -> SCX_DSQ_LOCAL_ON | cpu"); + if (status != SCX_TEST_PASS) + return status; + + status = run_scenario(skel, true, + "user DSQ -> scx_bpf_dsq_move_to_local()"); + if (status != SCX_TEST_PASS) + return status; + + return SCX_TEST_PASS; +} + +static void cleanup(void *ctx) +{ + struct dequeue_remote *skel = ctx; + + dequeue_remote__destroy(skel); +} + +struct scx_test dequeue_remote_test = { + .name = "dequeue_remote", + .description = "Verify ops.dequeue() on SCX-internal cross-CPU migrations", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; + +REGISTER_SCX_TEST(&dequeue_remote_test)