mirror of
https://github.com/torvalds/linux.git
synced 2026-09-23 22:14:03 +02:00
Merge branch 'for-7.2-fixes' into for-7.3
Pull to receive:db4e9defd2("sched_ext: Record an error on errno-only sub-enable failure")49b3378a75("sched_ext: Fix premature ops->priv publication in scx_alloc_and_add_sched()")e6979d05c6("tools/sched_ext: scx - Fix cmask_subset(), cmask_equal() and cmask_weight()") for further sub-sched changes and to resolve the conflicts with the sub-sched updates on for-7.3.db4e9defd2adds scx_error() to the sub-enable err_disable sink which for-7.3 moved from ext.c into sub.c. Resolved by applying the fix to scx_sub_enable_workfn() in sub.c.49b3378a75drops RCU_INIT_POINTER() from an scx_alloc_and_add_sched() unwind label whose body changed with for-7.3's stall_cpus addition. Resolved by dropping the line from the updated unwind. Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
commit
ad45691d8c
|
|
@ -493,8 +493,9 @@ a freshly woken up task gets on a CPU.
|
|||
Where to Look
|
||||
=============
|
||||
|
||||
* ``include/linux/sched/ext.h`` defines the core data structures, ops table
|
||||
and constants.
|
||||
* ``include/linux/sched/ext.h`` defines the core data structures and
|
||||
constants, while the ops table (``struct sched_ext_ops``) is defined in
|
||||
``kernel/sched/ext/internal.h``.
|
||||
|
||||
* ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers.
|
||||
The functions prefixed with ``scx_bpf_`` can be called from the BPF
|
||||
|
|
@ -555,7 +556,8 @@ ABI Instability
|
|||
===============
|
||||
|
||||
The APIs provided by sched_ext to BPF schedulers programs have no stability
|
||||
guarantees. This includes the ops table callbacks and constants defined in
|
||||
guarantees. This includes the ops table callbacks defined in
|
||||
``kernel/sched/ext/internal.h`` and the constants defined in
|
||||
``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
|
||||
``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``.
|
||||
|
||||
|
|
|
|||
|
|
@ -389,6 +389,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
|
|||
*/
|
||||
DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
|
||||
|
||||
static void switch_rq_lock(struct rq *from, struct rq *to)
|
||||
{
|
||||
bool tracked = scx_locked_rq() == from;
|
||||
|
||||
if (tracked)
|
||||
update_locked_rq(NULL);
|
||||
raw_spin_rq_unlock(from);
|
||||
raw_spin_rq_lock(to);
|
||||
if (tracked)
|
||||
update_locked_rq(to);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flipped on enable per sch->is_cid_type. Declared in internal.h so
|
||||
* subsystem inlines can read it.
|
||||
|
|
@ -2099,8 +2111,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
|
|||
deactivate_task(src_rq, p, 0);
|
||||
set_task_cpu(p, cpu_of(dst_rq));
|
||||
|
||||
raw_spin_rq_unlock(src_rq);
|
||||
raw_spin_rq_lock(dst_rq);
|
||||
switch_rq_lock(src_rq, dst_rq);
|
||||
|
||||
/*
|
||||
* We want to pass scx-specific enq_flags but activate_task() will
|
||||
|
|
@ -2433,9 +2444,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
|
|||
|
||||
/* switch to @src_rq lock */
|
||||
if (locked_rq != src_rq) {
|
||||
raw_spin_rq_unlock(locked_rq);
|
||||
switch_rq_lock(locked_rq, src_rq);
|
||||
locked_rq = src_rq;
|
||||
raw_spin_rq_lock(src_rq);
|
||||
}
|
||||
|
||||
/* task_rq couldn't have changed if we're still the holding cpu */
|
||||
|
|
@ -2469,10 +2479,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
|
|||
}
|
||||
|
||||
/* switch back to @rq lock */
|
||||
if (locked_rq != rq) {
|
||||
raw_spin_rq_unlock(locked_rq);
|
||||
raw_spin_rq_lock(rq);
|
||||
}
|
||||
if (locked_rq != rq)
|
||||
switch_rq_lock(locked_rq, rq);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2694,24 +2702,38 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
|
|||
|
||||
/*
|
||||
* @p is getting newly scheduled or got kicked after someone updated its
|
||||
* slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
|
||||
* slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
|
||||
* stopped. See scx_can_stop_tick().
|
||||
*
|
||||
* Moreover, refresh the load_avgs just when transitioning in and out of
|
||||
* nohz. In the future, we might want to add a mechanism to update
|
||||
* load_avgs periodically on tick-stopped CPUs.
|
||||
*/
|
||||
if ((p->scx.slice == SCX_SLICE_INF) !=
|
||||
(bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
|
||||
if (p->scx.slice == SCX_SLICE_INF)
|
||||
if (p->scx.slice == SCX_SLICE_INF) {
|
||||
if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
|
||||
/*
|
||||
* Bypass mode always assigns finite slices, so @p
|
||||
* can't have an infinite slice while bypassing.
|
||||
* Therefore, sched_update_tick_dependency() can safely
|
||||
* evaluate the outgoing task.
|
||||
*/
|
||||
rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
|
||||
else
|
||||
rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
|
||||
sched_update_tick_dependency(rq);
|
||||
|
||||
sched_update_tick_dependency(rq);
|
||||
update_other_load_avgs(rq);
|
||||
}
|
||||
} else {
|
||||
if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
|
||||
rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
|
||||
update_other_load_avgs(rq);
|
||||
}
|
||||
|
||||
/*
|
||||
* For now, let's refresh the load_avgs just when transitioning
|
||||
* in and out of nohz. In the future, we might want to add a
|
||||
* mechanism which calls the following periodically on
|
||||
* tick-stopped CPUs.
|
||||
* @rq still references the outgoing scheduling context. A finite
|
||||
* slice is sufficient by itself to require the tick.
|
||||
*/
|
||||
update_other_load_avgs(rq);
|
||||
if (tick_nohz_full_cpu(cpu_of(rq)))
|
||||
tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4035,6 +4057,15 @@ bool scx_can_stop_tick(struct rq *rq)
|
|||
if (p->sched_class != &ext_sched_class)
|
||||
return true;
|
||||
|
||||
/*
|
||||
* @rq->curr may still reference an outgoing EXT task after it has been
|
||||
* dequeued. If no EXT tasks are accounted on @rq, ignore its stale
|
||||
* slice state. If another task is dispatched from a DSQ,
|
||||
* set_next_task_scx() will update the dependency for the incoming task.
|
||||
*/
|
||||
if (!rq->scx.nr_running)
|
||||
return true;
|
||||
|
||||
if (scx_bypassing(sch, cpu_of(rq)))
|
||||
return false;
|
||||
|
||||
|
|
@ -6363,11 +6394,6 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
|
|||
sch->ops = *cmd->ops;
|
||||
}
|
||||
|
||||
rcu_assign_pointer(ops->priv, sch);
|
||||
|
||||
sch->kobj.kset = scx_kset;
|
||||
INIT_LIST_HEAD(&sch->all);
|
||||
|
||||
#ifdef CONFIG_EXT_SUB_SCHED
|
||||
char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
|
||||
if (!buf) {
|
||||
|
|
@ -6385,7 +6411,19 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
|
|||
sch->cgrp = cgrp;
|
||||
INIT_LIST_HEAD(&sch->children);
|
||||
INIT_LIST_HEAD(&sch->sibling);
|
||||
#endif /* CONFIG_EXT_SUB_SCHED */
|
||||
|
||||
/*
|
||||
* Publishing makes @sch visible to scx_prog_sched() readers. Failure
|
||||
* paths after this point must free @sch through kobject_put() whose
|
||||
* release path defers the actual freeing by an RCU grace period.
|
||||
*/
|
||||
rcu_assign_pointer(ops->priv, sch);
|
||||
|
||||
sch->kobj.kset = scx_kset;
|
||||
INIT_LIST_HEAD(&sch->all);
|
||||
|
||||
#ifdef CONFIG_EXT_SUB_SCHED
|
||||
if (parent) {
|
||||
/*
|
||||
* Pin @parent for @sch's lifetime. The kobject hierarchy pins
|
||||
|
|
@ -6440,7 +6478,6 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
|
|||
|
||||
#ifdef CONFIG_EXT_SUB_SCHED
|
||||
err_free_lb_resched:
|
||||
RCU_INIT_POINTER(ops->priv, NULL);
|
||||
free_cpumask_var(sch->stall_cpus);
|
||||
#endif
|
||||
err_free_lb_resched_cpumask:
|
||||
|
|
@ -7988,10 +8025,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
|
|||
in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
|
||||
|
||||
if (in_balance) {
|
||||
if (this_rq != src_rq) {
|
||||
raw_spin_rq_unlock(this_rq);
|
||||
raw_spin_rq_lock(src_rq);
|
||||
}
|
||||
if (this_rq != src_rq)
|
||||
switch_rq_lock(this_rq, src_rq);
|
||||
} else {
|
||||
raw_spin_rq_lock(src_rq);
|
||||
}
|
||||
|
|
@ -8023,10 +8058,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
|
|||
dispatched = true;
|
||||
out:
|
||||
if (in_balance) {
|
||||
if (this_rq != locked_rq) {
|
||||
raw_spin_rq_unlock(locked_rq);
|
||||
raw_spin_rq_lock(this_rq);
|
||||
}
|
||||
if (this_rq != locked_rq)
|
||||
switch_rq_lock(locked_rq, this_rq);
|
||||
} else {
|
||||
raw_spin_rq_unlock_irqrestore(locked_rq, flags);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -570,6 +570,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
|
|||
percpu_up_write(&scx_fork_rwsem);
|
||||
err_disable:
|
||||
mutex_unlock(&scx_enable_mutex);
|
||||
/*
|
||||
* Some enable failures only return an errno (e.g. -ENOMEM from an
|
||||
* allocation) without calling scx_error(). Record it so
|
||||
* scx_flush_disable_work() runs the disable and ops.exit() fires.
|
||||
*/
|
||||
scx_error(sch, "scx_sub_enable() failed (%d)", ret);
|
||||
scx_flush_disable_work(sch);
|
||||
cmd->ret = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
|
|||
|
||||
if (a->base != b->base || a->nr_cids != b->nr_cids)
|
||||
return false;
|
||||
nr_words = CMASK_NR_WORDS(a->nr_cids);
|
||||
if (a->nr_cids == 0)
|
||||
return true;
|
||||
nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
|
||||
|
||||
bpf_for(i, 0, CMASK_MAX_WORDS) {
|
||||
if (i >= nr_words)
|
||||
|
|
@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
|
|||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* True iff every bit set in @a is also set in @b over the intersection of
|
||||
* their ranges. Bits of @a outside @b's range fail the test.
|
||||
*/
|
||||
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
|
||||
const struct scx_cmask __arena *b)
|
||||
{
|
||||
u32 a_end = a->base + a->nr_cids;
|
||||
u32 b_end = b->base + b->nr_cids;
|
||||
u32 a_wbase = a->base / 64;
|
||||
u32 b_wbase = b->base / 64;
|
||||
u32 nr_words, i;
|
||||
|
||||
/* any bit of @a outside @b's range is a subset violation */
|
||||
if (a->base < b->base || a_end > b_end)
|
||||
return false;
|
||||
|
||||
nr_words = CMASK_NR_WORDS(a->nr_cids);
|
||||
bpf_for(i, 0, CMASK_MAX_WORDS) {
|
||||
u32 wi_b;
|
||||
|
||||
if (i >= nr_words)
|
||||
break;
|
||||
wi_b = a_wbase + i - b_wbase;
|
||||
if (a->bits[i] & ~b->bits[wi_b])
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* cmask_next_set - find the first set bit at or after @cid
|
||||
* @m: cmask to search
|
||||
|
|
@ -488,16 +460,66 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
|
|||
(cid) < (m)->base + (m)->nr_cids; \
|
||||
(cid) = cmask_next_set((m), (cid) + 1))
|
||||
|
||||
/*
|
||||
* True iff every bit set in @a is also set in @b. Matches the kernel-side
|
||||
* scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside
|
||||
* @b's range fail the test.
|
||||
*/
|
||||
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
|
||||
const struct scx_cmask __arena *b)
|
||||
{
|
||||
u32 a_end = a->base + a->nr_cids;
|
||||
u32 b_end = b->base + b->nr_cids;
|
||||
u32 a_wbase = a->base / 64;
|
||||
u32 b_wbase = b->base / 64;
|
||||
u32 lo = a->base > b->base ? a->base : b->base;
|
||||
u32 hi = a_end < b_end ? a_end : b_end;
|
||||
u32 lo_word, hi_word, i;
|
||||
|
||||
/* set bits of @a outside @b's range can't be in @b */
|
||||
if (a->base < b->base &&
|
||||
cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end))
|
||||
return false;
|
||||
if (a_end > b_end &&
|
||||
cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end)
|
||||
return false;
|
||||
|
||||
if (lo >= hi)
|
||||
return true;
|
||||
|
||||
/*
|
||||
* Walk the words the range intersection spans. Plain word tests
|
||||
* suffice: the scans above guarantee @a has no set bit outside @b's
|
||||
* range and padding bits are kept clear by all cmask helpers.
|
||||
*/
|
||||
lo_word = lo / 64;
|
||||
hi_word = (hi - 1) / 64;
|
||||
|
||||
bpf_for(i, 0, CMASK_MAX_WORDS) {
|
||||
u32 w = lo_word + i;
|
||||
|
||||
if (w > hi_word)
|
||||
break;
|
||||
if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase])
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Population count over [base, base + nr_cids). Padding bits in the head/tail
|
||||
* words are guaranteed zero by the mutating helpers, so a flat popcount over
|
||||
* all words is correct.
|
||||
* the words the range spans is correct.
|
||||
*/
|
||||
static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
|
||||
{
|
||||
u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
|
||||
u32 nr_words, i;
|
||||
u32 count = 0;
|
||||
|
||||
if (!m->nr_cids)
|
||||
return 0;
|
||||
nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
|
||||
|
||||
bpf_for(i, 0, CMASK_MAX_WORDS) {
|
||||
if (i >= nr_words)
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ auto-test-targets := \
|
|||
maybe_null \
|
||||
minimal \
|
||||
non_scx_kfunc_deny \
|
||||
nohz_tick \
|
||||
numa \
|
||||
allowed_cpus \
|
||||
peek_dsq \
|
||||
|
|
|
|||
65
tools/testing/selftests/sched_ext/nohz_tick.bpf.c
Normal file
65
tools/testing/selftests/sched_ext/nohz_tick.bpf.c
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
|
||||
*
|
||||
* Exercise tick dependency transitions between infinite and finite slices.
|
||||
*/
|
||||
#include <scx/common.bpf.h>
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
||||
const volatile s32 test_cpu;
|
||||
bool finite_phase;
|
||||
u64 nr_inf_running;
|
||||
u64 nr_finite_running;
|
||||
u64 nr_finite_ticks;
|
||||
|
||||
UEI_DEFINE(uei);
|
||||
|
||||
s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu,
|
||||
u64 wake_flags)
|
||||
{
|
||||
return prev_cpu;
|
||||
}
|
||||
|
||||
void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags)
|
||||
{
|
||||
u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF;
|
||||
|
||||
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags);
|
||||
if (enq_flags & SCX_ENQ_LAST)
|
||||
scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE);
|
||||
}
|
||||
|
||||
void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p)
|
||||
{
|
||||
if (bpf_get_smp_processor_id() != test_cpu)
|
||||
return;
|
||||
|
||||
if (finite_phase)
|
||||
__sync_fetch_and_add(&nr_finite_running, 1);
|
||||
else
|
||||
__sync_fetch_and_add(&nr_inf_running, 1);
|
||||
}
|
||||
|
||||
void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p)
|
||||
{
|
||||
if (bpf_get_smp_processor_id() == test_cpu && finite_phase)
|
||||
__sync_fetch_and_add(&nr_finite_ticks, 1);
|
||||
}
|
||||
|
||||
void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei)
|
||||
{
|
||||
UEI_RECORD(uei, ei);
|
||||
}
|
||||
|
||||
SEC(".struct_ops.link")
|
||||
struct sched_ext_ops nohz_tick_ops = {
|
||||
.select_cpu = (void *)nohz_tick_select_cpu,
|
||||
.enqueue = (void *)nohz_tick_enqueue,
|
||||
.running = (void *)nohz_tick_running,
|
||||
.tick = (void *)nohz_tick_tick,
|
||||
.exit = (void *)nohz_tick_exit,
|
||||
.name = "nohz_tick",
|
||||
.timeout_ms = 1000U,
|
||||
};
|
||||
347
tools/testing/selftests/sched_ext/nohz_tick.c
Normal file
347
tools/testing/selftests/sched_ext/nohz_tick.c
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
|
||||
*
|
||||
* Validate that a finite-slice EXT task restarts the scheduler tick when it
|
||||
* follows an infinite-slice EXT task and an idle interval on a NOHZ_FULL CPU.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
|
||||
#include <bpf/bpf.h>
|
||||
#include <errno.h>
|
||||
#include <sched.h>
|
||||
#include <signal.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/prctl.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <scx/common.h>
|
||||
|
||||
#include "nohz_tick.bpf.skel.h"
|
||||
#include "scx_test.h"
|
||||
|
||||
#ifndef SCHED_EXT
|
||||
#define SCHED_EXT 7
|
||||
#endif
|
||||
|
||||
#define MIN_FINITE_TICKS 3
|
||||
#define PHASE_TIMEOUT_MS 1000
|
||||
|
||||
struct nohz_tick_ctx {
|
||||
struct nohz_tick *skel;
|
||||
cpu_set_t original_mask;
|
||||
int test_cpu;
|
||||
};
|
||||
|
||||
static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
|
||||
{
|
||||
int cpu;
|
||||
|
||||
for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++)
|
||||
if (CPU_ISSET(cpu, mask))
|
||||
return cpu;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int find_nohz_full_cpu(const cpu_set_t *allowed)
|
||||
{
|
||||
char buf[4096], *cur, *end;
|
||||
FILE *file;
|
||||
|
||||
file = fopen("/sys/devices/system/cpu/nohz_full", "r");
|
||||
if (!file)
|
||||
return -1;
|
||||
if (!fgets(buf, sizeof(buf), file)) {
|
||||
fclose(file);
|
||||
return -1;
|
||||
}
|
||||
fclose(file);
|
||||
|
||||
cur = buf;
|
||||
while (*cur) {
|
||||
long first, last;
|
||||
int cpu;
|
||||
|
||||
while (*cur == ' ' || *cur == '\t' || *cur == ',')
|
||||
cur++;
|
||||
if (*cur < '0' || *cur > '9')
|
||||
break;
|
||||
|
||||
errno = 0;
|
||||
first = strtol(cur, &end, 10);
|
||||
if (errno || end == cur || first < 0 || first >= CPU_SETSIZE)
|
||||
return -1;
|
||||
cur = end;
|
||||
last = first;
|
||||
if (*cur == '-') {
|
||||
cur++;
|
||||
errno = 0;
|
||||
last = strtol(cur, &end, 10);
|
||||
if (errno || end == cur || last < first)
|
||||
return -1;
|
||||
cur = end;
|
||||
}
|
||||
|
||||
cpu = first_allowed_cpu(allowed, first, last);
|
||||
if (cpu >= 0)
|
||||
return cpu;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
static pid_t start_worker(int cpu)
|
||||
{
|
||||
struct sched_param param = {};
|
||||
cpu_set_t mask;
|
||||
pid_t parent;
|
||||
pid_t pid;
|
||||
|
||||
parent = getpid();
|
||||
pid = fork();
|
||||
if (pid != 0)
|
||||
return pid;
|
||||
if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent)
|
||||
_exit(1);
|
||||
|
||||
/*
|
||||
* Become EXT before touching the target so it stays idle until wakeup.
|
||||
*/
|
||||
if (sched_setscheduler(0, SCHED_EXT, ¶m))
|
||||
_exit(1);
|
||||
|
||||
CPU_ZERO(&mask);
|
||||
CPU_SET(cpu, &mask);
|
||||
if (sched_setaffinity(0, sizeof(mask), &mask))
|
||||
_exit(1);
|
||||
|
||||
for (;;)
|
||||
asm volatile("" ::: "memory");
|
||||
}
|
||||
|
||||
static void stop_worker(pid_t pid)
|
||||
{
|
||||
if (pid <= 0)
|
||||
return;
|
||||
|
||||
kill(pid, SIGKILL);
|
||||
waitpid(pid, NULL, 0);
|
||||
}
|
||||
|
||||
static int pause_worker(pid_t pid)
|
||||
{
|
||||
int status;
|
||||
|
||||
if (kill(pid, SIGSTOP))
|
||||
return -errno;
|
||||
if (waitpid(pid, &status, WUNTRACED) != pid)
|
||||
return -errno;
|
||||
if (!WIFSTOPPED(status))
|
||||
return -ECHILD;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms)
|
||||
{
|
||||
int elapsed;
|
||||
|
||||
for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
|
||||
if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value)
|
||||
return true;
|
||||
usleep(1000);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static enum scx_test_status setup(void **ctx_ptr)
|
||||
{
|
||||
struct nohz_tick_ctx *ctx;
|
||||
cpu_set_t controller_mask;
|
||||
int cpu;
|
||||
|
||||
ctx = calloc(1, sizeof(*ctx));
|
||||
SCX_FAIL_IF(!ctx, "Failed to allocate context");
|
||||
if (sched_getaffinity(0, sizeof(ctx->original_mask),
|
||||
&ctx->original_mask)) {
|
||||
free(ctx);
|
||||
SCX_FAIL("Failed to get affinity (%d)", errno);
|
||||
}
|
||||
|
||||
cpu = find_nohz_full_cpu(&ctx->original_mask);
|
||||
if (cpu < 0) {
|
||||
fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n");
|
||||
free(ctx);
|
||||
return SCX_TEST_SKIP;
|
||||
}
|
||||
|
||||
controller_mask = ctx->original_mask;
|
||||
CPU_CLR(cpu, &controller_mask);
|
||||
if (CPU_COUNT(&controller_mask) == 0) {
|
||||
fprintf(stderr, "SKIP: no housekeeping CPU available\n");
|
||||
free(ctx);
|
||||
return SCX_TEST_SKIP;
|
||||
}
|
||||
|
||||
ctx->test_cpu = cpu;
|
||||
ctx->skel = nohz_tick__open();
|
||||
if (!ctx->skel) {
|
||||
free(ctx);
|
||||
SCX_FAIL("Failed to open skeleton");
|
||||
}
|
||||
|
||||
SCX_ENUM_INIT(ctx->skel);
|
||||
ctx->skel->rodata->test_cpu = cpu;
|
||||
ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL |
|
||||
SCX_OPS_ENQ_LAST;
|
||||
if (nohz_tick__load(ctx->skel)) {
|
||||
nohz_tick__destroy(ctx->skel);
|
||||
free(ctx);
|
||||
SCX_FAIL("Failed to load skeleton");
|
||||
}
|
||||
|
||||
if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) {
|
||||
nohz_tick__destroy(ctx->skel);
|
||||
free(ctx);
|
||||
SCX_FAIL("Failed to move controller off CPU %d (%d)", cpu, errno);
|
||||
}
|
||||
|
||||
*ctx_ptr = ctx;
|
||||
return SCX_TEST_PASS;
|
||||
}
|
||||
|
||||
static enum scx_test_status run(void *ctx_ptr)
|
||||
{
|
||||
struct nohz_tick_ctx *ctx = ctx_ptr;
|
||||
struct nohz_tick *skel = ctx->skel;
|
||||
struct bpf_link *link = NULL;
|
||||
enum scx_test_status status = SCX_TEST_FAIL;
|
||||
pid_t finite_worker = -1;
|
||||
pid_t inf_worker = -1;
|
||||
u64 finite_running;
|
||||
u64 finite_ticks;
|
||||
int ret;
|
||||
|
||||
link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops);
|
||||
if (!link) {
|
||||
SCX_ERR("Failed to attach scheduler");
|
||||
goto out;
|
||||
}
|
||||
|
||||
/*
|
||||
* Establish SCX_RQ_CAN_STOP_TICK with an infinite-slice task.
|
||||
*/
|
||||
inf_worker = start_worker(ctx->test_cpu);
|
||||
if (inf_worker < 0) {
|
||||
SCX_ERR("Failed to start infinite-slice worker (%d)", errno);
|
||||
goto out;
|
||||
}
|
||||
if (!wait_for_counter(&skel->bss->nr_inf_running, 1,
|
||||
PHASE_TIMEOUT_MS)) {
|
||||
SCX_ERR("Infinite-slice worker was not scheduled");
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Block without exiting so the rq retains the infinite-slice state. */
|
||||
ret = pause_worker(inf_worker);
|
||||
if (ret) {
|
||||
SCX_ERR("Failed to stop infinite-slice worker (%d)", ret);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Let the target enter idle with its tick stopped. */
|
||||
usleep(100000);
|
||||
|
||||
/*
|
||||
* The next EXT task receives a finite slice and must restart the tick.
|
||||
*/
|
||||
__atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE);
|
||||
finite_worker = start_worker(ctx->test_cpu);
|
||||
if (finite_worker < 0) {
|
||||
SCX_ERR("Failed to start finite-slice worker (%d)", errno);
|
||||
goto out;
|
||||
}
|
||||
if (!wait_for_counter(&skel->bss->nr_finite_running, 1,
|
||||
PHASE_TIMEOUT_MS)) {
|
||||
SCX_ERR("Finite-slice worker was not scheduled");
|
||||
goto out;
|
||||
}
|
||||
if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
|
||||
PHASE_TIMEOUT_MS)) {
|
||||
SCX_ERR("Finite-slice worker received only %llu scheduler ticks",
|
||||
(unsigned long long)skel->bss->nr_finite_ticks);
|
||||
goto out;
|
||||
}
|
||||
stop_worker(finite_worker);
|
||||
finite_worker = -1;
|
||||
|
||||
/*
|
||||
* Leave the CPU idle after a finite-slice task. The next finite-slice
|
||||
* task must restart the tick even though the slice type is unchanged.
|
||||
*/
|
||||
usleep(100000);
|
||||
finite_running = __atomic_load_n(&skel->bss->nr_finite_running,
|
||||
__ATOMIC_RELAXED);
|
||||
finite_ticks = __atomic_load_n(&skel->bss->nr_finite_ticks,
|
||||
__ATOMIC_RELAXED);
|
||||
|
||||
finite_worker = start_worker(ctx->test_cpu);
|
||||
if (finite_worker < 0) {
|
||||
SCX_ERR("Failed to start second finite-slice worker (%d)", errno);
|
||||
goto out;
|
||||
}
|
||||
if (!wait_for_counter(&skel->bss->nr_finite_running,
|
||||
finite_running + 1, PHASE_TIMEOUT_MS)) {
|
||||
SCX_ERR("Second finite-slice worker was not scheduled");
|
||||
goto out;
|
||||
}
|
||||
if (!wait_for_counter(&skel->bss->nr_finite_ticks,
|
||||
finite_ticks + MIN_FINITE_TICKS,
|
||||
PHASE_TIMEOUT_MS)) {
|
||||
SCX_ERR("Second finite-slice worker received only %llu scheduler ticks",
|
||||
(unsigned long long)(skel->bss->nr_finite_ticks -
|
||||
finite_ticks));
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
|
||||
SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
|
||||
(unsigned long long)skel->data->uei.kind,
|
||||
(long long)skel->data->uei.exit_code);
|
||||
goto out;
|
||||
}
|
||||
|
||||
fprintf(stderr, "CPU %d received %llu finite-slice ticks\n",
|
||||
ctx->test_cpu,
|
||||
(unsigned long long)skel->bss->nr_finite_ticks);
|
||||
status = SCX_TEST_PASS;
|
||||
out:
|
||||
stop_worker(finite_worker);
|
||||
stop_worker(inf_worker);
|
||||
if (link)
|
||||
bpf_link__destroy(link);
|
||||
return status;
|
||||
}
|
||||
|
||||
static void cleanup(void *ctx_ptr)
|
||||
{
|
||||
struct nohz_tick_ctx *ctx = ctx_ptr;
|
||||
|
||||
sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask);
|
||||
nohz_tick__destroy(ctx->skel);
|
||||
free(ctx);
|
||||
}
|
||||
|
||||
struct scx_test nohz_tick = {
|
||||
.name = "nohz_tick",
|
||||
.description = "Verify finite EXT slices restart the NOHZ_FULL tick",
|
||||
.setup = setup,
|
||||
.run = run,
|
||||
.cleanup = cleanup,
|
||||
};
|
||||
REGISTER_SCX_TEST(&nohz_tick)
|
||||
Loading…
Reference in New Issue
Block a user