github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

net/sched: annotate data-races around sch->qstats.backlog

Browse Source 
Add qstats_backlog_sub() and qstats_backlog_add() helpers
and use them instead of open-coding them.

These helpers use WRITE_ONCE() to prevent store-tearing.

Also use WRITE_ONCE() in fq_reset() and qdisc_reset()
when sch->qstats.backlog is cleared.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Toke Høiland-Jørgensen <toke@toke.dk>
Link: https://patch.msgid.link/20260510091455.4039245-4-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet 2026-05-10 09:14:50 +00:00 committed by Jakub Kicinski
parent 863cd78652
commit 0e5b05f44d
21 changed files with 41 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/net/sch_generic.h
View File

@ -965,10 +965,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch,
bstats_update(&sch->bstats, skb);
}
static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val)
{
WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val);
}
static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,
const struct sk_buff *skb)
{
sch->qstats.backlog -= qdisc_pkt_len(skb);
qstats_backlog_sub(sch, qdisc_pkt_len(skb));
}
static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
@ -977,10 +982,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
}
static inline void qstats_backlog_add(struct Qdisc *sch, u32 val)
{
WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val);
}
static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,
const struct sk_buff *skb)
{
sch->qstats.backlog += qdisc_pkt_len(skb);
qstats_backlog_add(sch, qdisc_pkt_len(skb));
}
static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
@ -1304,7 +1314,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
qdisc_qstats_cpu_qlen_inc(sch);
this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
} else {
sch->qstats.backlog += pkt_len;
qstats_backlog_add(sch, pkt_len);
qdisc_qlen_inc(sch);
}
}

2
net/sched/sch_api.c
View File

@ -806,7 +806,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
cops->qlen_notify(sch, cl);
}
WRITE_ONCE(sch->q.qlen, sch->q.qlen - n);
sch->qstats.backlog -= len;
qstats_backlog_sub(sch, len);
__qdisc_qstats_drop(sch, drops);
}
rcu_read_unlock();

10
net/sched/sch_cake.c
View File

@ -1600,10 +1600,10 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
b->unresponsive_flow_count + 1);
len = qdisc_pkt_len(skb);
q->buffer_used -= skb->truesize;
qstats_backlog_sub(sch, len);
q->buffer_used -= skb->truesize;
WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] - len);
sch->qstats.backlog -= len;
WRITE_ONCE(flow->dropped, flow->dropped + 1);
WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1);
@ -1830,7 +1830,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
/* stats */
sch->qstats.backlog += slen;
qstats_backlog_add(sch, slen);
q->avg_window_bytes += slen;
WRITE_ONCE(b->bytes, b->bytes + slen);
WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen);
@ -1867,7 +1867,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
/* stats */
WRITE_ONCE(b->packets, b->packets + 1);
sch->qstats.backlog += len - ack_pkt_len;
qstats_backlog_add(sch, len - ack_pkt_len);
q->avg_window_bytes += len - ack_pkt_len;
WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len);
WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len);
@ -1985,7 +1985,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch)
len = qdisc_pkt_len(skb);
WRITE_ONCE(b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len);
WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
sch->qstats.backlog -= len;
qstats_backlog_sub(sch, len);
q->buffer_used -= skb->truesize;
qdisc_qlen_dec(sch);

2
net/sched/sch_cbs.c
View File

@ -96,7 +96,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (err != NET_XMIT_SUCCESS)
return err;
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;

2
net/sched/sch_codel.c
View File

@ -42,7 +42,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
if (skb) {
sch->qstats.backlog -= qdisc_pkt_len(skb);
qstats_backlog_sub(sch, qdisc_pkt_len(skb));
prefetch(&skb->end); /* we'll need skb_shinfo() */
}
return skb;

2
net/sched/sch_drr.c
View File

@ -365,7 +365,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch,
cl->deficit = cl->quantum;
}
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return err;
}

2
net/sched/sch_ets.c
View File

@ -448,7 +448,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
cl->deficit = cl->quantum;
}
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return err;
}

2
net/sched/sch_fq.c
View File

@ -802,7 +802,7 @@ static void fq_reset(struct Qdisc *sch)
unsigned int idx;
WRITE_ONCE(sch->q.qlen, 0);
sch->qstats.backlog = 0;
WRITE_ONCE(sch->qstats.backlog, 0);
fq_flow_purge(&q->internal);

4
net/sched/sch_fq_codel.c
View File

@ -177,7 +177,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] - len);
q->memory_usage -= mem;
__qdisc_qstats_drop(sch, i);
sch->qstats.backlog -= len;
qstats_backlog_sub(sch, len);
WRITE_ONCE(sch->q.qlen, sch->q.qlen - i);
return idx;
}
@ -268,7 +268,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
q->backlogs[flow - q->flows] - qdisc_pkt_len(skb));
q->memory_usage -= get_codel_cb(skb)->mem_usage;
qdisc_qlen_dec(sch);
sch->qstats.backlog -= qdisc_pkt_len(skb);
qdisc_qstats_backlog_dec(sch, skb);
}
return skb;
}

4
net/sched/sch_fq_pie.c
View File

@ -184,7 +184,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
pkt_len = qdisc_pkt_len(skb);
q->stats.packets_in++;
q->memory_usage += skb->truesize;
sch->qstats.backlog += pkt_len;
qstats_backlog_add(sch, pkt_len);
qdisc_qlen_inc(sch);
flow_queue_add(sel_flow, skb);
if (list_empty(&sel_flow->flowchain)) {
@ -262,7 +262,7 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)
if (flow->head) {
skb = dequeue_head(flow);
pkt_len = qdisc_pkt_len(skb);
sch->qstats.backlog -= pkt_len;
qstats_backlog_sub(sch, pkt_len);
qdisc_qlen_dec(sch);
qdisc_bstats_update(sch, skb);
}

2
net/sched/sch_generic.c
View File

@ -1063,7 +1063,7 @@ void qdisc_reset(struct Qdisc *qdisc)
__skb_queue_purge(&qdisc->skb_bad_txq);
WRITE_ONCE(qdisc->q.qlen, 0);
qdisc->qstats.backlog = 0;
WRITE_ONCE(qdisc->qstats.backlog, 0);
}
EXPORT_SYMBOL(qdisc_reset);

2
net/sched/sch_gred.c
View File

@ -388,7 +388,7 @@ static int gred_offload_dump_stats(struct Qdisc *sch)
bytes += u64_stats_read(&hw_stats->stats.bstats[i].bytes);
packets += u64_stats_read(&hw_stats->stats.bstats[i].packets);
sch->qstats.qlen += hw_stats->stats.qstats[i].qlen;
sch->qstats.backlog += hw_stats->stats.qstats[i].backlog;
qstats_backlog_add(sch, hw_stats->stats.qstats[i].backlog);
__qdisc_qstats_drop(sch, hw_stats->stats.qstats[i].drops);
sch->qstats.requeues += hw_stats->stats.qstats[i].requeues;
sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits;

2
net/sched/sch_hfsc.c
View File

@ -1560,7 +1560,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
return err;
}
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
if (first && !cl_in_el_or_vttree(cl)) {

2
net/sched/sch_htb.c
View File

@ -650,7 +650,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
htb_activate(q, cl);
}
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}

2
net/sched/sch_netem.c
View File

@ -750,7 +750,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
if (err != NET_XMIT_SUCCESS) {
if (net_xmit_drop_count(err))
qdisc_qstats_drop(sch);
sch->qstats.backlog -= pkt_len;
qstats_backlog_sub(sch, pkt_len);
qdisc_qlen_dec(sch);
qdisc_tree_reduce_backlog(sch, 1, pkt_len);
}

2
net/sched/sch_prio.c
View File

@ -85,7 +85,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
ret = qdisc_enqueue(skb, qdisc, to_free);
if (ret == NET_XMIT_SUCCESS) {
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}

2
net/sched/sch_qfq.c
View File

@ -1264,7 +1264,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
_bstats_update(&cl->bstats, len, gso_segs);
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
agg = cl->agg;

2
net/sched/sch_red.c
View File

@ -138,7 +138,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
len = qdisc_pkt_len(skb);
ret = qdisc_enqueue(skb, child, to_free);
if (likely(ret == NET_XMIT_SUCCESS)) {
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
} else if (net_xmit_drop_count(ret)) {
WRITE_ONCE(q->stats.pdrop,

4
net/sched/sch_sfb.c
View File

@ -415,7 +415,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
memcpy(&cb, sfb_skb_cb(skb), sizeof(cb));
ret = qdisc_enqueue(skb, child, to_free);
if (likely(ret == NET_XMIT_SUCCESS)) {
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
increment_qlen(&cb, q);
} else if (net_xmit_drop_count(ret)) {
@ -592,7 +592,7 @@ static int sfb_dump(struct Qdisc *sch, struct sk_buff *skb)
.penalty_burst = q->penalty_burst,
};
sch->qstats.backlog = q->qdisc->qstats.backlog;
WRITE_ONCE(sch->qstats.backlog, READ_ONCE(q->qdisc->qstats.backlog));
opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
if (opts == NULL)
goto nla_put_failure;

2
net/sched/sch_sfq.c
View File

@ -427,7 +427,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
/* We know we have at least one packet in queue */
head = slot_dequeue_head(slot);
delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
sch->qstats.backlog -= delta;
qstats_backlog_sub(sch, delta);
WRITE_ONCE(slot->backlog, slot->backlog - delta);
qdisc_drop_reason(head, sch, to_free, QDISC_DROP_FLOW_LIMIT);

4
net/sched/sch_tbf.c
View File

@ -232,7 +232,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch,
}
}
WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb);
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
if (nb > 0) {
qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len);
consume_skb(skb);
@ -263,7 +263,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
return ret;
}
sch->qstats.backlog += len;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}