github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-24 06:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched_ext: Use READ_ONCE/WRITE_ONCE in cmask word ops and drop _RACY variants

Browse Source 
The cmask ops can operate on BPF-arena cmasks which BPF programs may read
and write concurrently. The _RACY op variants existed to make such lockless
reads explicit but this turned out to be too restrictive. Mark the word
accesses in all the two-cmask ops with READ_ONCE/WRITE_ONCE instead and drop
the _RACY variants.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
This commit is contained in:
Tejun Heo 2026-07-09 12:06:10 -10:00
parent ad45691d8c
commit 05e72aeaaa
2 changed files with 10 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
kernel/sched/ext/cid.c
View File

@ -395,17 +395,15 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux)
* bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero
* @dst bits that lie outside @src's range.
*
* The _RACY variants are otherwise identical to their non-racy counterpart but
* read @src word-by-word via data_race(). Memory ordering with concurrent
* writers is the caller's responsibility.
* Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src
* locklessly. Memory ordering against concurrent writers is the caller's
* responsibility.
*/
enum cmask_op2 {
/* mutating */
CMASK_OP2_AND,
CMASK_OP2_OR,
CMASK_OP2_OR_RACY,
CMASK_OP2_COPY,
CMASK_OP2_COPY_RACY,
CMASK_OP2_ANDNOT,
/* predicates - short-circuit when the per-word result is true */
CMASK_OP2_SUBSET,
@ -422,28 +420,22 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask,
{
switch (op) {
case CMASK_OP2_AND:
*av &= ~mask | *bp;
WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp)));
return false;
case CMASK_OP2_OR:
*av |= *bp & mask;
return false;
case CMASK_OP2_OR_RACY:
*av |= data_race(*bp) & mask;
WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_COPY:
*av = (*av & ~mask) | (*bp & mask);
return false;
case CMASK_OP2_COPY_RACY:
*av = (*av & ~mask) | (data_race(*bp) & mask);
WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_ANDNOT:
*av &= ~(*bp & mask);
WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_SUBSET:
/* stop on the first bit in @sub not set in @super */
return (*bp & ~*av) & mask;
return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask;
case CMASK_OP2_INTERSECTS:
return (*av & *bp) & mask;
return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask;
}
unreachable();
}
@ -504,7 +496,7 @@ static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask,
{
switch (op) {
case CMASK_OP1_ANY_SET:
return *ap & mask;
return READ_ONCE(*ap) & mask;
}
unreachable();
}
@ -556,39 +548,12 @@ void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src)
src->bits, src->base, src->nr_cids, CMASK_OP2_OR);
}
/**
* scx_cmask_or_racy - OR @src into @dst, reading @src without locking
*
* @src is read word-by-word through data_race(). Same per-bit independence
* rationale as scx_cmask_copy_racy(). Memory ordering with writers is the
* caller's responsibility.
*/
void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY);
}
void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_COPY);
}
/**
* scx_cmask_copy_racy - Snapshot @src into @dst without locking
*
* @src is read word-by-word through data_race(). Head/tail masking matches
* scx_cmask_copy(). Each bit in a cmask is independent, so partial updates
* just leave some bits fresher than others. Memory ordering with writers is
* the caller's responsibility.
*/
void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY);
}
void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,

2
kernel/sched/ext/cid.h
View File

@ -57,9 +57,7 @@ void scx_cmask_clear(struct scx_cmask *m);
void scx_cmask_fill(struct scx_cmask *m);
void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src);
bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super);
bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b);