github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 17:47:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'net-sched-prepare-lockless-qdisc-dumps'

Browse Source 
Eric Dumazet says:

====================
net/sched: prepare lockless qdisc dumps

Goal is to no longer acquire RTNL in qdisc dumps.

This series annotate data-races, and change mq and mq_prio to
no longer acquire children qdisc spinlocks.
====================

Link: https://patch.msgid.link/20260510091455.4039245-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2026-05-11 18:28:11 -07:00
commit 89cc3ffe5d
33 changed files with 256 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/net/gen_stats.h
View File

@ -47,19 +47,19 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type,
int padattr);
int gnet_stats_copy_basic(struct gnet_dump *d,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b, bool running);
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b, bool running);
void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b, bool running);
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b, bool running);
int gnet_stats_copy_basic_hw(struct gnet_dump *d,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b, bool running);
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b, bool running);
int gnet_stats_copy_rate_est(struct gnet_dump *d,
struct net_rate_estimator __rcu **ptr);
int gnet_stats_copy_queue(struct gnet_dump *d,
struct gnet_stats_queue __percpu *cpu_q,
struct gnet_stats_queue *q, __u32 qlen);
const struct gnet_stats_queue __percpu *cpu_q,
const struct gnet_stats_queue *q, __u32 qlen);
void gnet_stats_add_queue(struct gnet_stats_queue *qstats,
const struct gnet_stats_queue __percpu *cpu_q,
const struct gnet_stats_queue *q);

64
include/net/sch_generic.h
View File

@ -542,6 +542,21 @@ static inline int qdisc_qlen(const struct Qdisc *q)
return q->q.qlen;
}
static inline int qdisc_qlen_lockless(const struct Qdisc *q)
{
return READ_ONCE(q->q.qlen);
}
static inline void qdisc_qlen_inc(struct Qdisc *q)
{
WRITE_ONCE(q->q.qlen, q->q.qlen + 1);
}
static inline void qdisc_qlen_dec(struct Qdisc *q)
{
WRITE_ONCE(q->q.qlen, q->q.qlen - 1);
}
static inline int qdisc_qlen_sum(const struct Qdisc *q)
{
__u32 qlen = q->qstats.qlen;
@ -549,9 +564,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q)
if (qdisc_is_percpu_stats(q)) {
for_each_possible_cpu(i)
qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen);
} else {
qlen += q->q.qlen;
qlen += qdisc_qlen_lockless(q);
}
return qlen;
@ -937,6 +952,15 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,
u64_stats_update_end(&bstats->syncp);
}
static inline void _bstats_set(struct gnet_stats_basic_sync *bstats,
u64 bytes, u64 packets)
{
u64_stats_update_begin(&bstats->syncp);
u64_stats_set(&bstats->bytes, bytes);
u64_stats_set(&bstats->packets, packets);
u64_stats_update_end(&bstats->syncp);
}
static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
const struct sk_buff *skb)
{
@ -955,10 +979,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch,
bstats_update(&sch->bstats, skb);
}
static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val)
{
WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val);
}
static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,
const struct sk_buff *skb)
{
sch->qstats.backlog -= qdisc_pkt_len(skb);
qstats_backlog_sub(sch, qdisc_pkt_len(skb));
}
static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
@ -967,10 +996,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
}
static inline void qstats_backlog_add(struct Qdisc *sch, u32 val)
{
WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val);
}
static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,
const struct sk_buff *skb)
{
sch->qstats.backlog += qdisc_pkt_len(skb);
qstats_backlog_add(sch, qdisc_pkt_len(skb));
}
static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
@ -1029,7 +1063,7 @@ static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
WRITE_ONCE(sch->qstats.overlimits, sch->qstats.overlimits + 1);
}
static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)
static inline int qdisc_qstats_copy(struct gnet_dump *d, const struct Qdisc *sch)
{
__u32 qlen = qdisc_qlen_sum(sch);
@ -1068,7 +1102,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
qh->tail = skb;
qh->head = skb;
}
qh->qlen++;
WRITE_ONCE(qh->qlen, qh->qlen + 1);
}
static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
@ -1086,7 +1120,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb,
if (!qh->head)
qh->tail = skb;
qh->head = skb;
qh->qlen++;
WRITE_ONCE(qh->qlen, qh->qlen + 1);
}
static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
@ -1095,7 +1129,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
if (likely(skb != NULL)) {
qh->head = skb->next;
qh->qlen--;
WRITE_ONCE(qh->qlen, qh->qlen - 1);
if (qh->head == NULL)
qh->tail = NULL;
skb->next = NULL;
@ -1110,7 +1144,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir
skb = __skb_dequeue(&sch->gso_skb);
if (skb) {
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
return skb;
}
@ -1266,7 +1300,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)
__skb_queue_head(&sch->gso_skb, skb);
/* it's still part of the queue */
qdisc_qstats_backlog_inc(sch, skb);
sch->q.qlen++;
qdisc_qlen_inc(sch);
}
}
@ -1283,7 +1317,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,
} else {
qdisc_qstats_backlog_dec(sch, skb);
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
}
}
@ -1294,8 +1328,8 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
qdisc_qstats_cpu_qlen_inc(sch);
this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
} else {
sch->qstats.backlog += pkt_len;
sch->q.qlen++;
qstats_backlog_add(sch, pkt_len);
qdisc_qlen_inc(sch);
}
}
@ -1311,7 +1345,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
qdisc_qstats_cpu_qlen_dec(sch);
} else {
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
}
} else {
skb = sch->dequeue(sch);
@ -1332,7 +1366,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
qh->head = NULL;
qh->tail = NULL;
qh->qlen = 0;
WRITE_ONCE(qh->qlen, 0);
}
}

51
net/core/gen_stats.c
View File

@ -124,13 +124,13 @@ void gnet_stats_basic_sync_init(struct gnet_stats_basic_sync *b)
EXPORT_SYMBOL(gnet_stats_basic_sync_init);
static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats,
struct gnet_stats_basic_sync __percpu *cpu)
const struct gnet_stats_basic_sync __percpu *cpu)
{
u64 t_bytes = 0, t_packets = 0;
int i;
for_each_possible_cpu(i) {
struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
unsigned int start;
u64 bytes, packets;
@ -147,8 +147,8 @@ static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats,
}
void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b, bool running)
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b, bool running)
{
unsigned int start;
u64 bytes = 0;
@ -172,8 +172,8 @@ void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
EXPORT_SYMBOL(gnet_stats_add_basic);
static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b, bool running)
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b, bool running)
{
unsigned int start;
@ -182,8 +182,7 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
int i;
for_each_possible_cpu(i) {
struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
unsigned int start;
const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
u64 bytes, packets;
do {
@ -209,8 +208,8 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
static int
___gnet_stats_copy_basic(struct gnet_dump *d,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b,
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b,
int type, bool running)
{
u64 bstats_bytes, bstats_packets;
@ -258,8 +257,8 @@ ___gnet_stats_copy_basic(struct gnet_dump *d,
*/
int
gnet_stats_copy_basic(struct gnet_dump *d,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b,
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b,
bool running)
{
return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC, running);
@ -285,8 +284,8 @@ EXPORT_SYMBOL(gnet_stats_copy_basic);
*/
int
gnet_stats_copy_basic_hw(struct gnet_dump *d,
struct gnet_stats_basic_sync __percpu *cpu,
struct gnet_stats_basic_sync *b,
const struct gnet_stats_basic_sync __percpu *cpu,
const struct gnet_stats_basic_sync *b,
bool running)
{
return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC_HW, running);
@ -345,11 +344,11 @@ static void gnet_stats_add_queue_cpu(struct gnet_stats_queue *qstats,
for_each_possible_cpu(i) {
const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i);
qstats->qlen += qcpu->qlen;
qstats->backlog += qcpu->backlog;
qstats->drops += qcpu->drops;
qstats->requeues += qcpu->requeues;
qstats->overlimits += qcpu->overlimits;
qstats->qlen += READ_ONCE(qcpu->qlen);
qstats->backlog += READ_ONCE(qcpu->backlog);
qstats->drops += READ_ONCE(qcpu->drops);
qstats->requeues += READ_ONCE(qcpu->requeues);
qstats->overlimits += READ_ONCE(qcpu->overlimits);
}
}
@ -360,11 +359,11 @@ void gnet_stats_add_queue(struct gnet_stats_queue *qstats,
if (cpu) {
gnet_stats_add_queue_cpu(qstats, cpu);
} else {
qstats->qlen += q->qlen;
qstats->backlog += q->backlog;
qstats->drops += q->drops;
qstats->requeues += q->requeues;
qstats->overlimits += q->overlimits;
qstats->qlen += READ_ONCE(q->qlen);
qstats->backlog += READ_ONCE(q->backlog);
qstats->drops += READ_ONCE(q->drops);
qstats->requeues += READ_ONCE(q->requeues);
qstats->overlimits += READ_ONCE(q->overlimits);
}
}
EXPORT_SYMBOL(gnet_stats_add_queue);
@ -385,8 +384,8 @@ EXPORT_SYMBOL(gnet_stats_add_queue);
*/
int
gnet_stats_copy_queue(struct gnet_dump *d,
struct gnet_stats_queue __percpu *cpu_q,
struct gnet_stats_queue *q, __u32 qlen)
const struct gnet_stats_queue __percpu *cpu_q,
const struct gnet_stats_queue *q, __u32 qlen)
{
struct gnet_stats_queue qstats = {0};

4
net/sched/sch_api.c
View File

@ -805,8 +805,8 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
cl = cops->find(sch, parentid);
cops->qlen_notify(sch, cl);
}
sch->q.qlen -= n;
sch->qstats.backlog -= len;
WRITE_ONCE(sch->q.qlen, sch->q.qlen - n);
qstats_backlog_sub(sch, len);
__qdisc_qstats_drop(sch, drops);
}
rcu_read_unlock();

18
net/sched/sch_cake.c
View File

@ -1600,10 +1600,10 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
b->unresponsive_flow_count + 1);
len = qdisc_pkt_len(skb);
q->buffer_used -= skb->truesize;
qstats_backlog_sub(sch, len);
q->buffer_used -= skb->truesize;
WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] - len);
sch->qstats.backlog -= len;
WRITE_ONCE(flow->dropped, flow->dropped + 1);
WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1);
@ -1612,7 +1612,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
cake_advance_shaper(q, b, skb, now, true);
qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT);
sch->q.qlen--;
qdisc_qlen_dec(sch);
cake_heapify(q, 0);
@ -1822,7 +1822,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
segs);
flow_queue_add(flow, segs);
sch->q.qlen++;
qdisc_qlen_inc(sch);
numsegs++;
slen += segs->len;
q->buffer_used += segs->truesize;
@ -1830,7 +1830,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
/* stats */
sch->qstats.backlog += slen;
qstats_backlog_add(sch, slen);
q->avg_window_bytes += slen;
WRITE_ONCE(b->bytes, b->bytes + slen);
WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen);
@ -1861,13 +1861,13 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len);
consume_skb(ack);
} else {
sch->q.qlen++;
qdisc_qlen_inc(sch);
q->buffer_used += skb->truesize;
}
/* stats */
WRITE_ONCE(b->packets, b->packets + 1);
sch->qstats.backlog += len - ack_pkt_len;
qstats_backlog_add(sch, len - ack_pkt_len);
q->avg_window_bytes += len - ack_pkt_len;
WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len);
WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len);
@ -1985,9 +1985,9 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch)
len = qdisc_pkt_len(skb);
WRITE_ONCE(b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len);
WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
sch->qstats.backlog -= len;
qstats_backlog_sub(sch, len);
q->buffer_used -= skb->truesize;
sch->q.qlen--;
qdisc_qlen_dec(sch);
if (q->overflow_timeout)
cake_heapify(q, b->overflow_idx[q->cur_flow]);

6
net/sched/sch_cbs.c
View File

@ -96,8 +96,8 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (err != NET_XMIT_SUCCESS)
return err;
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child)
qdisc_qstats_backlog_dec(sch, skb);
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

8
net/sched/sch_choke.c
View File

@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx,
if (idx == q->tail)
choke_zap_tail_holes(q);
--sch->q.qlen;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
qdisc_drop(skb, sch, to_free);
@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (sch->q.qlen < q->limit) {
q->tab[q->tail] = skb;
q->tail = (q->tail + 1) & q->tab_mask;
++sch->q.qlen;
qdisc_qlen_inc(sch);
qdisc_qstats_backlog_inc(sch, skb);
return NET_XMIT_SUCCESS;
}
@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch)
skb = q->tab[q->head];
q->tab[q->head] = NULL;
choke_zap_head_holes(q);
--sch->q.qlen;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
qdisc_bstats_update(sch, skb);
@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt,
}
dropped += qdisc_pkt_len(skb);
qdisc_qstats_backlog_dec(sch, skb);
--sch->q.qlen;
qdisc_qlen_dec(sch);
rtnl_qdisc_drop(skb, sch);
}
qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped);

2
net/sched/sch_codel.c
View File

@ -42,7 +42,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
if (skb) {
sch->qstats.backlog -= qdisc_pkt_len(skb);
qstats_backlog_sub(sch, qdisc_pkt_len(skb));
prefetch(&skb->end); /* we'll need skb_shinfo() */
}
return skb;

6
net/sched/sch_drr.c
View File

@ -365,8 +365,8 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch,
cl->deficit = cl->quantum;
}
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return err;
}
@ -399,7 +399,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch)
bstats_update(&cl->bstats, skb);
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

6
net/sched/sch_dualpi2.c
View File

@ -415,7 +415,7 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch,
dualpi2_skb_cb(skb)->apply_step = skb_apply_step(skb, q);
/* Keep the overall qdisc stats consistent */
++sch->q.qlen;
qdisc_qlen_inc(sch);
qdisc_qstats_backlog_inc(sch, skb);
++q->packets_in_l;
if (!q->l_head_ts)
@ -530,7 +530,7 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch,
qdisc_qstats_backlog_dec(q->l_queue, skb);
/* Keep the global queue size consistent */
--sch->q.qlen;
qdisc_qlen_dec(sch);
q->memory_used -= skb->truesize;
} else if (c_len) {
skb = __qdisc_dequeue_head(&sch->q);
@ -888,7 +888,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt,
* l_queue on enqueue; qdisc_dequeue_internal()
* handled l_queue, so we further account for sch.
*/
--sch->q.qlen;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
q->memory_used -= skb->truesize;
rtnl_qdisc_drop(skb, q->l_queue);

8
net/sched/sch_etf.c
View File

@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch,
rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost);
qdisc_qstats_backlog_inc(sch, nskb);
sch->q.qlen++;
qdisc_qlen_inc(sch);
/* Now we may need to re-arm the qdisc watchdog for the next packet. */
reset_watchdog(sch);
@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb,
qdisc_qstats_backlog_dec(sch, skb);
qdisc_drop(skb, sch, &to_free);
qdisc_qstats_overlimit(sch);
sch->q.qlen--;
qdisc_qlen_dec(sch);
}
kfree_skb_list(to_free);
@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb)
q->last = skb->tstamp;
sch->q.qlen--;
qdisc_qlen_dec(sch);
}
static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch)
@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch)
rb_erase_cached(&skb->rbnode, &q->head);
rtnl_kfree_skbs(skb, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
}
}

6
net/sched/sch_ets.c
View File

@ -448,8 +448,8 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
cl->deficit = cl->quantum;
}
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return err;
}
@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb)
{
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

8
net/sched/sch_fq.c
View File

@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow,
fq_erase_head(sch, flow, skb);
skb_mark_not_on_list(skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_bstats_update(sch, skb);
}
@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
flow_queue_add(f, skb);
qdisc_qstats_backlog_inc(sch, skb);
sch->q.qlen++;
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
@ -801,8 +801,8 @@ static void fq_reset(struct Qdisc *sch)
struct fq_flow *f;
unsigned int idx;
sch->q.qlen = 0;
sch->qstats.backlog = 0;
WRITE_ONCE(sch->q.qlen, 0);
WRITE_ONCE(sch->qstats.backlog, 0);
fq_flow_purge(&q->internal);

11
net/sched/sch_fq_codel.c
View File

@ -177,8 +177,8 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] - len);
q->memory_usage -= mem;
__qdisc_qstats_drop(sch, i);
sch->qstats.backlog -= len;
sch->q.qlen -= i;
qstats_backlog_sub(sch, len);
WRITE_ONCE(sch->q.qlen, sch->q.qlen - i);
return idx;
}
@ -215,7 +215,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
get_codel_cb(skb)->mem_usage = skb->truesize;
q->memory_usage += get_codel_cb(skb)->mem_usage;
memory_limited = q->memory_usage > q->memory_limit;
if (++sch->q.qlen <= sch->limit && !memory_limited)
qdisc_qlen_inc(sch);
if (sch->q.qlen <= sch->limit && !memory_limited)
return NET_XMIT_SUCCESS;
prev_backlog = sch->qstats.backlog;
@ -266,8 +267,8 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
WRITE_ONCE(q->backlogs[flow - q->flows],
q->backlogs[flow - q->flows] - qdisc_pkt_len(skb));
q->memory_usage -= get_codel_cb(skb)->mem_usage;
sch->q.qlen--;
sch->qstats.backlog -= qdisc_pkt_len(skb);
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
}
return skb;
}

8
net/sched/sch_fq_pie.c
View File

@ -184,8 +184,8 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
pkt_len = qdisc_pkt_len(skb);
q->stats.packets_in++;
q->memory_usage += skb->truesize;
sch->qstats.backlog += pkt_len;
sch->q.qlen++;
qstats_backlog_add(sch, pkt_len);
qdisc_qlen_inc(sch);
flow_queue_add(sel_flow, skb);
if (list_empty(&sel_flow->flowchain)) {
list_add_tail(&sel_flow->flowchain, &q->new_flows);
@ -262,8 +262,8 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)
if (flow->head) {
skb = dequeue_head(flow);
pkt_len = qdisc_pkt_len(skb);
sch->qstats.backlog -= pkt_len;
sch->q.qlen--;
qstats_backlog_sub(sch, pkt_len);
qdisc_qlen_dec(sch);
qdisc_bstats_update(sch, skb);
}

12
net/sched/sch_generic.c
View File

@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
qdisc_qstats_cpu_qlen_dec(q);
} else {
qdisc_qstats_backlog_dec(q, skb);
q->q.qlen--;
qdisc_qlen_dec(q);
}
} else {
skb = SKB_XOFF_MAGIC;
@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
qdisc_qstats_cpu_qlen_inc(q);
} else {
qdisc_qstats_backlog_inc(q, skb);
q->q.qlen++;
qdisc_qlen_inc(q);
}
if (lock)
@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
} else {
q->qstats.requeues++;
qdisc_qstats_backlog_inc(q, skb);
q->q.qlen++;
qdisc_qlen_inc(q);
}
skb = next;
@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
qdisc_qstats_cpu_qlen_dec(q);
} else {
qdisc_qstats_backlog_dec(q, skb);
q->q.qlen--;
qdisc_qlen_dec(q);
}
} else {
skb = NULL;
@ -1062,8 +1062,8 @@ void qdisc_reset(struct Qdisc *qdisc)
__skb_queue_purge(&qdisc->gso_skb);
__skb_queue_purge(&qdisc->skb_bad_txq);
qdisc->q.qlen = 0;
qdisc->qstats.backlog = 0;
WRITE_ONCE(qdisc->q.qlen, 0);
WRITE_ONCE(qdisc->qstats.backlog, 0);
}
EXPORT_SYMBOL(qdisc_reset);

2
net/sched/sch_gred.c
View File

@ -388,7 +388,7 @@ static int gred_offload_dump_stats(struct Qdisc *sch)
bytes += u64_stats_read(&hw_stats->stats.bstats[i].bytes);
packets += u64_stats_read(&hw_stats->stats.bstats[i].packets);
sch->qstats.qlen += hw_stats->stats.qstats[i].qlen;
sch->qstats.backlog += hw_stats->stats.qstats[i].backlog;
qstats_backlog_add(sch, hw_stats->stats.qstats[i].backlog);
__qdisc_qstats_drop(sch, hw_stats->stats.qstats[i].drops);
sch->qstats.requeues += hw_stats->stats.qstats[i].requeues;
sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits;

6
net/sched/sch_hfsc.c
View File

@ -1560,8 +1560,8 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
return err;
}
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
if (first && !cl_in_el_or_vttree(cl)) {
if (cl->cl_flags & HFSC_RSC)
@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch)
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

7
net/sched/sch_hhf.c
View File

@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free)
if (bucket->head) {
struct sk_buff *skb = dequeue_head(bucket);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
qdisc_drop(skb, sch, to_free);
}
@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
bucket->deficit = weight * q->quantum;
}
if (++sch->q.qlen <= sch->limit)
qdisc_qlen_inc(sch);
if (sch->q.qlen <= sch->limit)
return NET_XMIT_SUCCESS;
prev_backlog = sch->qstats.backlog;
@ -443,7 +444,7 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch)
if (bucket->head) {
skb = dequeue_head(bucket);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
}

6
net/sched/sch_htb.c
View File

@ -650,8 +650,8 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
htb_activate(q, cl);
}
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
@ -951,7 +951,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch)
ok:
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

37
net/sched/sch_mq.c
View File

@ -143,30 +143,43 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL");
void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb)
{
struct net_device *dev = qdisc_dev(sch);
struct Qdisc *qdisc;
struct gnet_stats_queue qstats = { 0 };
struct gnet_stats_basic_sync bstats;
const struct Qdisc *qdisc;
unsigned int qlen = 0;
unsigned int ntx;
sch->q.qlen = 0;
gnet_stats_basic_sync_init(&sch->bstats);
memset(&sch->qstats, 0, sizeof(sch->qstats));
gnet_stats_basic_sync_init(&bstats);
/* MQ supports lockless qdiscs. However, statistics accounting needs
* to account for all, none, or a mix of locked and unlocked child
* qdiscs. Percpu stats are added to counters in-band and locking
* qdisc totals are added at end.
*/
rcu_read_lock();
for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
qdisc = rtnl_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
spin_lock_bh(qdisc_lock(qdisc));
qdisc = rcu_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
gnet_stats_add_basic(&sch->bstats, qdisc->cpu_bstats,
&qdisc->bstats, false);
gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats,
gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
&qdisc->bstats, true);
gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
&qdisc->qstats);
sch->q.qlen += qdisc_qlen(qdisc);
spin_unlock_bh(qdisc_lock(qdisc));
qlen += qdisc_qlen_lockless(qdisc);
}
rcu_read_unlock();
spin_lock_bh(qdisc_lock(sch));
_bstats_set(&sch->bstats, u64_stats_read(&bstats.bytes),
u64_stats_read(&bstats.packets));
spin_unlock_bh(qdisc_lock(sch));
WRITE_ONCE(sch->qstats.qlen, qstats.qlen);
WRITE_ONCE(sch->qstats.backlog, qstats.backlog);
WRITE_ONCE(sch->qstats.drops, qstats.drops);
WRITE_ONCE(sch->qstats.requeues, qstats.requeues);
WRITE_ONCE(sch->qstats.overlimits, qstats.overlimits);
WRITE_ONCE(sch->q.qlen, qlen);
}
EXPORT_SYMBOL_NS_GPL(mq_dump_common, "NET_SCHED_INTERNAL");

93
net/sched/sch_mqprio.c
View File

@ -551,34 +551,47 @@ static int mqprio_dump_tc_entries(struct mqprio_sched *priv,
static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb)
{
struct net_device *dev = qdisc_dev(sch);
struct mqprio_sched *priv = qdisc_priv(sch);
struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb);
struct mqprio_sched *priv = qdisc_priv(sch);
struct net_device *dev = qdisc_dev(sch);
struct gnet_stats_queue qstats = { 0 };
struct gnet_stats_basic_sync bstats;
struct tc_mqprio_qopt opt = { 0 };
struct Qdisc *qdisc;
const struct Qdisc *qdisc;
unsigned int qlen = 0;
unsigned int ntx;
sch->q.qlen = 0;
gnet_stats_basic_sync_init(&sch->bstats);
memset(&sch->qstats, 0, sizeof(sch->qstats));
gnet_stats_basic_sync_init(&bstats);
/* MQ supports lockless qdiscs. However, statistics accounting needs
* to account for all, none, or a mix of locked and unlocked child
* qdiscs. Percpu stats are added to counters in-band and locking
* qdisc totals are added at end.
*/
rcu_read_lock();
for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
qdisc = rtnl_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
spin_lock_bh(qdisc_lock(qdisc));
qdisc = rcu_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
gnet_stats_add_basic(&sch->bstats, qdisc->cpu_bstats,
&qdisc->bstats, false);
gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats,
gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
&qdisc->bstats, true);
gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
&qdisc->qstats);
sch->q.qlen += qdisc_qlen(qdisc);
spin_unlock_bh(qdisc_lock(qdisc));
qlen += qdisc_qlen_lockless(qdisc);
}
rcu_read_unlock();
spin_lock_bh(qdisc_lock(sch));
_bstats_set(&sch->bstats, u64_stats_read(&bstats.bytes),
u64_stats_read(&bstats.packets));
spin_unlock_bh(qdisc_lock(sch));
WRITE_ONCE(sch->qstats.qlen, qstats.qlen);
WRITE_ONCE(sch->qstats.backlog, qstats.backlog);
WRITE_ONCE(sch->qstats.drops, qstats.drops);
WRITE_ONCE(sch->qstats.requeues, qstats.requeues);
WRITE_ONCE(sch->qstats.overlimits, qstats.overlimits);
WRITE_ONCE(sch->q.qlen, qlen);
mqprio_qopt_reconstruct(dev, &opt);
opt.hw = priv->hw_offload;
@ -659,58 +672,48 @@ static int mqprio_dump_class(struct Qdisc *sch, unsigned long cl,
static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
struct gnet_dump *d)
__releases(d->lock)
__acquires(d->lock)
{
const struct Qdisc *qdisc;
int res = 0;
rcu_read_lock();
if (cl >= TC_H_MIN_PRIORITY) {
int i;
__u32 qlen;
struct gnet_stats_queue qstats = {0};
struct gnet_stats_basic_sync bstats;
struct net_device *dev = qdisc_dev(sch);
struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK];
struct gnet_stats_queue qstats = {0};
struct gnet_stats_basic_sync bstats;
u32 qlen = 0;
int i;
gnet_stats_basic_sync_init(&bstats);
/* Drop lock here it will be reclaimed before touching
* statistics this is required because the d->lock we
* hold here is the look on dev_queue->qdisc_sleeping
* also acquired below.
*/
if (d->lock)
spin_unlock_bh(d->lock);
for (i = tc.offset; i < tc.offset + tc.count; i++) {
struct netdev_queue *q = netdev_get_tx_queue(dev, i);
struct Qdisc *qdisc = rtnl_dereference(q->qdisc);
spin_lock_bh(qdisc_lock(qdisc));
qdisc = rcu_dereference(q->qdisc);
gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
&qdisc->bstats, false);
&qdisc->bstats, true);
gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
&qdisc->qstats);
sch->q.qlen += qdisc_qlen(qdisc);
spin_unlock_bh(qdisc_lock(qdisc));
qlen += qdisc_qlen_lockless(qdisc);
}
qlen = qdisc_qlen(sch) + qstats.qlen;
/* Reclaim root sleeping lock before completing stats */
if (d->lock)
spin_lock_bh(d->lock);
qlen = qlen + qstats.qlen;
if (gnet_stats_copy_basic(d, NULL, &bstats, false) < 0 ||
gnet_stats_copy_queue(d, NULL, &qstats, qlen) < 0)
return -1;
res = -1;
} else {
struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl);
sch = rtnl_dereference(dev_queue->qdisc_sleeping);
if (gnet_stats_copy_basic(d, sch->cpu_bstats,
&sch->bstats, true) < 0 ||
qdisc_qstats_copy(d, sch) < 0)
return -1;
qdisc = rcu_dereference(dev_queue->qdisc_sleeping);
if (gnet_stats_copy_basic(d, qdisc->cpu_bstats,
&qdisc->bstats, true) < 0 ||
qdisc_qstats_copy(d, qdisc) < 0)
res = -1;
}
return 0;
rcu_read_unlock();
return res;
}
static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)

4
net/sched/sch_multiq.c
View File

@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
ret = qdisc_enqueue(skb, qdisc, to_free);
if (ret == NET_XMIT_SUCCESS) {
sch->q.qlen++;
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
if (net_xmit_drop_count(ret))
@ -106,7 +106,7 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch)
skb = qdisc->dequeue(qdisc);
if (skb) {
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}
}

12
net/sched/sch_netem.c
View File

@ -416,7 +416,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)
rb_insert_color(&nskb->rbnode, &q->t_root);
}
q->t_len++;
sch->q.qlen++;
qdisc_qlen_inc(sch);
}
/* netem can't properly corrupt a megapacket (like we get from GSO), so instead
@ -750,20 +750,20 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
if (err != NET_XMIT_SUCCESS) {
if (net_xmit_drop_count(err))
qdisc_qstats_drop(sch);
sch->qstats.backlog -= pkt_len;
sch->q.qlen--;
qstats_backlog_sub(sch, pkt_len);
qdisc_qlen_dec(sch);
qdisc_tree_reduce_backlog(sch, 1, pkt_len);
}
goto tfifo_dequeue;
}
sch->q.qlen--;
qdisc_qlen_dec(sch);
goto deliver;
}
if (q->qdisc) {
skb = q->qdisc->ops->dequeue(q->qdisc);
if (skb) {
sch->q.qlen--;
qdisc_qlen_dec(sch);
goto deliver;
}
}
@ -776,7 +776,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
if (q->qdisc) {
skb = q->qdisc->ops->dequeue(q->qdisc);
if (skb) {
sch->q.qlen--;
qdisc_qlen_dec(sch);
goto deliver;
}
}

6
net/sched/sch_prio.c
View File

@ -85,8 +85,8 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
ret = qdisc_enqueue(skb, qdisc, to_free);
if (ret == NET_XMIT_SUCCESS) {
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
if (net_xmit_drop_count(ret))
@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch)
if (skb) {
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}
}

8
net/sched/sch_qfq.c
View File

@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch)
if (!skb)
return NULL;
sch->q.qlen--;
qdisc_qlen_dec(sch);
skb = agg_dequeue(in_serv_agg, cl, len);
if (!skb) {
sch->q.qlen++;
qdisc_qlen_inc(sch);
return NULL;
}
@ -1264,8 +1264,8 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
_bstats_update(&cl->bstats, len, gso_segs);
sch->qstats.backlog += len;
++sch->q.qlen;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
agg = cl->agg;
/* if the class is active, then done here */

6
net/sched/sch_red.c
View File

@ -138,8 +138,8 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
len = qdisc_pkt_len(skb);
ret = qdisc_enqueue(skb, child, to_free);
if (likely(ret == NET_XMIT_SUCCESS)) {
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
} else if (net_xmit_drop_count(ret)) {
WRITE_ONCE(q->stats.pdrop,
q->stats.pdrop + 1);
@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch)
if (skb) {
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
} else {
if (!red_is_idling(&q->vars))
red_start_of_idle_period(&q->vars);

8
net/sched/sch_sfb.c
View File

@ -415,8 +415,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
memcpy(&cb, sfb_skb_cb(skb), sizeof(cb));
ret = qdisc_enqueue(skb, child, to_free);
if (likely(ret == NET_XMIT_SUCCESS)) {
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
increment_qlen(&cb, q);
} else if (net_xmit_drop_count(ret)) {
WRITE_ONCE(q->stats.childdrop,
@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch)
if (skb) {
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
decrement_qlen(skb, q);
}
@ -592,7 +592,7 @@ static int sfb_dump(struct Qdisc *sch, struct sk_buff *skb)
.penalty_burst = q->penalty_burst,
};
sch->qstats.backlog = q->qdisc->qstats.backlog;
WRITE_ONCE(sch->qstats.backlog, READ_ONCE(q->qdisc->qstats.backlog));
opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
if (opts == NULL)
goto nla_put_failure;

11
net/sched/sch_sfq.c
View File

@ -302,7 +302,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
len = qdisc_pkt_len(skb);
WRITE_ONCE(slot->backlog, slot->backlog - len);
sfq_dec(q, x);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT);
return len;
@ -427,7 +427,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
/* We know we have at least one packet in queue */
head = slot_dequeue_head(slot);
delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
sch->qstats.backlog -= delta;
qstats_backlog_sub(sch, delta);
WRITE_ONCE(slot->backlog, slot->backlog - delta);
qdisc_drop_reason(head, sch, to_free, QDISC_DROP_FLOW_LIMIT);
@ -456,7 +456,8 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
/* We could use a bigger initial quantum for new flows */
WRITE_ONCE(slot->allot, q->quantum);
}
if (++sch->q.qlen <= q->limit)
qdisc_qlen_inc(sch);
if (sch->q.qlen <= q->limit)
return NET_XMIT_SUCCESS;
qlen = slot->qlen;
@ -497,7 +498,7 @@ sfq_dequeue(struct Qdisc *sch)
skb = slot_dequeue_head(slot);
sfq_dec(q, a);
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb));
/* Is the slot empty? */
@ -596,7 +597,7 @@ static void sfq_rehash(struct Qdisc *sch)
WRITE_ONCE(slot->allot, q->quantum);
}
}
sch->q.qlen -= dropped;
WRITE_ONCE(sch->q.qlen, sch->q.qlen - dropped);
qdisc_tree_reduce_backlog(sch, dropped, drop_len);
}

4
net/sched/sch_skbprio.c
View File

@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (prio < q->lowest_prio)
q->lowest_prio = prio;
sch->q.qlen++;
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch)
if (unlikely(!skb))
return NULL;
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_qstats_backlog_dec(sch, skb);
qdisc_bstats_update(sch, skb);

4
net/sched/sch_taprio.c
View File

@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
}
qdisc_qstats_backlog_inc(sch, skb);
sch->q.qlen++;
qdisc_qlen_inc(sch);
return qdisc_enqueue(skb, child, to_free);
}
@ -755,7 +755,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
return skb;
}

10
net/sched/sch_tbf.c
View File

@ -231,8 +231,8 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch,
len += seg_len;
}
}
sch->q.qlen += nb;
sch->qstats.backlog += len;
WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb);
qstats_backlog_add(sch, len);
if (nb > 0) {
qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len);
consume_skb(skb);
@ -263,8 +263,8 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
return ret;
}
sch->qstats.backlog += len;
sch->q.qlen++;
qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
return NET_XMIT_SUCCESS;
}
@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch)
q->tokens = toks;
q->ptokens = ptoks;
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_qlen_dec(sch);
qdisc_bstats_update(sch, skb);
return skb;
}

2
net/sched/sch_teql.c
View File

@ -107,7 +107,7 @@ teql_dequeue(struct Qdisc *sch)
} else {
qdisc_bstats_update(sch, skb);
}
sch->q.qlen = dat->q.qlen + q->q.qlen;
WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen));
return skb;
}