github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-30 10:04:04 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'net_sched-speedup-qdisc-dequeue'

Browse Source 
Eric Dumazet says:

====================
net_sched: speedup qdisc dequeue

Avoid up to two cache line misses in qdisc dequeue() to fetch
skb_shinfo(skb)->gso_segs/gso_size while qdisc spinlock is held.

Idea is to cache gso_segs at enqueue time before spinlock is
acquired, in the first skb cache line, where we already
have qdisc_skb_cb(skb)->pkt_len.

This series gives a 8 % improvement in a TX intensive workload.

(120 Mpps -> 130 Mpps on a Turin host, IDPF with 32 TX queues)

v2: https://lore.kernel.org/netdev/20251111093204.1432437-1-edumazet@google.com/
v1: https://lore.kernel.org/netdev/20251110094505.3335073-1-edumazet@google.com/T/#m8f562ed148f807c02fd02c6cd243604d449615b9
====================

Link: https://patch.msgid.link/20251121083256.674562-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni 2025-11-25 16:10:34 +01:00
commit 61e628023d
15 changed files with 147 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/net/pkt_sched.h
View File

@ -114,12 +114,13 @@ bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
void __qdisc_run(struct Qdisc *q);
static inline void qdisc_run(struct Qdisc *q)
static inline struct sk_buff *qdisc_run(struct Qdisc *q)
{
if (qdisc_run_begin(q)) {
__qdisc_run(q);
qdisc_run_end(q);
return qdisc_run_end(q);
}
return NULL;
}
extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];

101
include/net/sch_generic.h
View File

@ -88,6 +88,8 @@ struct Qdisc {
#define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */
#define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */
#define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */
#define TCQ_F_DEQUEUE_DROPS 0x400 /* ->dequeue() can drop packets in q->to_free */
u32 limit;
const struct Qdisc_ops *ops;
struct qdisc_size_table __rcu *stab;
@ -103,17 +105,26 @@ struct Qdisc {
int pad;
refcount_t refcnt;
/*
* For performance sake on SMP, we put highly modified fields at the end
*/
struct sk_buff_head gso_skb ____cacheline_aligned_in_smp;
struct qdisc_skb_head q;
struct gnet_stats_basic_sync bstats;
struct gnet_stats_queue qstats;
bool running; /* must be written under qdisc spinlock */
unsigned long state;
struct Qdisc *next_sched;
struct sk_buff_head skb_bad_txq;
/* Cache line potentially dirtied in dequeue() or __netif_reschedule(). */
__cacheline_group_begin(Qdisc_read_mostly) ____cacheline_aligned;
struct sk_buff_head gso_skb;
struct Qdisc *next_sched;
struct sk_buff_head skb_bad_txq;
__cacheline_group_end(Qdisc_read_mostly);
/* Fields dirtied in dequeue() fast path. */
__cacheline_group_begin(Qdisc_write) ____cacheline_aligned;
struct qdisc_skb_head q;
unsigned long state;
struct gnet_stats_basic_sync bstats;
bool running; /* must be written under qdisc spinlock */
/* Note : we only change qstats.backlog in fast path. */
struct gnet_stats_queue qstats;
struct sk_buff *to_free;
__cacheline_group_end(Qdisc_write);
atomic_long_t defer_count ____cacheline_aligned_in_smp;
struct llist_head defer_list;
@ -211,8 +222,10 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
return true;
}
static inline void qdisc_run_end(struct Qdisc *qdisc)
static inline struct sk_buff *qdisc_run_end(struct Qdisc *qdisc)
{
struct sk_buff *to_free = NULL;
if (qdisc->flags & TCQ_F_NOLOCK) {
spin_unlock(&qdisc->seqlock);
@ -225,9 +238,16 @@ static inline void qdisc_run_end(struct Qdisc *qdisc)
if (unlikely(test_bit(__QDISC_STATE_MISSED,
&qdisc->state)))
__netif_schedule(qdisc);
} else {
WRITE_ONCE(qdisc->running, false);
return NULL;
}
if (qdisc->flags & TCQ_F_DEQUEUE_DROPS) {
to_free = qdisc->to_free;
if (to_free)
qdisc->to_free = NULL;
}
WRITE_ONCE(qdisc->running, false);
return to_free;
}
static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
@ -429,13 +449,16 @@ struct tcf_proto {
};
struct qdisc_skb_cb {
struct {
unsigned int pkt_len;
u16 slave_dev_queue_mapping;
u16 tc_classid;
};
unsigned int pkt_len;
u16 pkt_segs;
u16 tc_classid;
#define QDISC_CB_PRIV_LEN 20
unsigned char data[QDISC_CB_PRIV_LEN];
u16 slave_dev_queue_mapping;
u8 post_ct:1;
u8 post_ct_snat:1;
u8 post_ct_dnat:1;
};
typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
@ -826,6 +849,15 @@ static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
return qdisc_skb_cb(skb)->pkt_len;
}
static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)
{
u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;
DEBUG_NET_WARN_ON_ONCE(pkt_segs !=
(skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1));
return pkt_segs;
}
/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
enum net_xmit_qdisc_t {
__NET_XMIT_STOLEN = 0x00010000,
@ -867,9 +899,7 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,
static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
const struct sk_buff *skb)
{
_bstats_update(bstats,
qdisc_pkt_len(skb),
skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1);
_bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));
}
static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,
@ -1064,11 +1094,8 @@ struct tc_skb_cb {
struct qdisc_skb_cb qdisc_cb;
u32 drop_reason;
u16 zone; /* Only valid if post_ct = true */
u16 zone; /* Only valid if qdisc_skb_cb(skb)->post_ct = true */
u16 mru;
u8 post_ct:1;
u8 post_ct_snat:1;
u8 post_ct_dnat:1;
};
static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
@ -1091,6 +1118,28 @@ static inline void tcf_set_drop_reason(const struct sk_buff *skb,
tc_skb_cb(skb)->drop_reason = reason;
}
static inline void tcf_kfree_skb_list(struct sk_buff *skb)
{
while (unlikely(skb)) {
struct sk_buff *next = skb->next;
prefetch(next);
kfree_skb_reason(skb, tcf_get_drop_reason(skb));
skb = next;
}
}
static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb,
enum skb_drop_reason reason)
{
DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS));
DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK);
tcf_set_drop_reason(skb, reason);
skb->next = q->to_free;
q->to_free = skb;
}
/* Instead of calling kfree_skb() while root qdisc lock is held,
* queue the skb for future freeing at end of __dev_xmit_skb()
*/

62
net/core/dev.c
View File

@ -4069,17 +4069,23 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d
}
EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
static void qdisc_pkt_len_init(struct sk_buff *skb)
static void qdisc_pkt_len_segs_init(struct sk_buff *skb)
{
const struct skb_shared_info *shinfo = skb_shinfo(skb);
struct skb_shared_info *shinfo = skb_shinfo(skb);
u16 gso_segs;
qdisc_skb_cb(skb)->pkt_len = skb->len;
if (!shinfo->gso_size) {
qdisc_skb_cb(skb)->pkt_segs = 1;
return;
}
qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs;
/* To get more precise estimation of bytes sent on wire,
* we add to pkt_len the headers size of all segments
*/
if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
u16 gso_segs = shinfo->gso_segs;
if (skb_transport_header_was_set(skb)) {
unsigned int hdr_len;
/* mac layer + network layer */
@ -4112,6 +4118,8 @@ static void qdisc_pkt_len_init(struct sk_buff *skb)
if (payload <= 0)
return;
gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
shinfo->gso_segs = gso_segs;
qdisc_skb_cb(skb)->pkt_segs = gso_segs;
}
qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
}
@ -4133,7 +4141,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
struct net_device *dev,
struct netdev_queue *txq)
{
struct sk_buff *next, *to_free = NULL;
struct sk_buff *next, *to_free = NULL, *to_free2 = NULL;
spinlock_t *root_lock = qdisc_lock(q);
struct llist_node *ll_list, *first_n;
unsigned long defer_count = 0;
@ -4152,9 +4160,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
if (unlikely(!nolock_qdisc_is_empty(q))) {
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
__qdisc_run(q);
qdisc_run_end(q);
to_free2 = qdisc_run_end(q);
goto no_lock_out;
goto free_skbs;
}
qdisc_bstats_cpu_update(q, skb);
@ -4162,18 +4170,14 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
!nolock_qdisc_is_empty(q))
__qdisc_run(q);
qdisc_run_end(q);
return NET_XMIT_SUCCESS;
to_free2 = qdisc_run_end(q);
rc = NET_XMIT_SUCCESS;
goto free_skbs;
}
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
qdisc_run(q);
no_lock_out:
if (unlikely(to_free))
kfree_skb_list_reason(to_free,
tcf_get_drop_reason(to_free));
return rc;
to_free2 = qdisc_run(q);
goto free_skbs;
}
/* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit.
@ -4186,7 +4190,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
do {
if (first_n && !defer_count) {
defer_count = atomic_long_inc_return(&q->defer_count);
if (unlikely(defer_count > q->limit)) {
if (unlikely(defer_count > READ_ONCE(q->limit))) {
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_DROP);
return NET_XMIT_DROP;
}
@ -4231,26 +4235,28 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
qdisc_bstats_update(q, skb);
if (sch_direct_xmit(skb, q, dev, txq, root_lock, true))
__qdisc_run(q);
qdisc_run_end(q);
to_free2 = qdisc_run_end(q);
rc = NET_XMIT_SUCCESS;
} else {
int count = 0;
llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
prefetch(next);
prefetch(&next->priority);
skb_mark_not_on_list(skb);
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
count++;
}
qdisc_run(q);
to_free2 = qdisc_run(q);
if (count != 1)
rc = NET_XMIT_SUCCESS;
}
unlock:
spin_unlock(root_lock);
if (unlikely(to_free))
kfree_skb_list_reason(to_free,
tcf_get_drop_reason(to_free));
free_skbs:
tcf_kfree_skb_list(to_free);
tcf_kfree_skb_list(to_free2);
return rc;
}
@ -4355,7 +4361,7 @@ static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
return ret;
tc_skb_cb(skb)->mru = 0;
tc_skb_cb(skb)->post_ct = false;
qdisc_skb_cb(skb)->post_ct = false;
tcf_set_drop_reason(skb, *drop_reason);
mini_qdisc_bstats_cpu_update(miniq, skb);
@ -4426,7 +4432,7 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
*pt_prev = NULL;
}
qdisc_skb_cb(skb)->pkt_len = skb->len;
qdisc_pkt_len_segs_init(skb);
tcx_set_ingress(skb, true);
if (static_branch_unlikely(&tcx_needed_key)) {
@ -4737,7 +4743,7 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
skb_update_prio(skb);
qdisc_pkt_len_init(skb);
qdisc_pkt_len_segs_init(skb);
tcx_set_ingress(skb, false);
#ifdef CONFIG_NET_EGRESS
if (static_branch_unlikely(&egress_needed_key)) {
@ -5743,8 +5749,9 @@ static __latent_entropy void net_tx_action(void)
rcu_read_lock();
while (head) {
struct Qdisc *q = head;
spinlock_t *root_lock = NULL;
struct sk_buff *to_free;
struct Qdisc *q = head;
head = head->next_sched;
@ -5771,9 +5778,10 @@ static __latent_entropy void net_tx_action(void)
}
clear_bit(__QDISC_STATE_SCHED, &q->state);
qdisc_run(q);
to_free = qdisc_run(q);
if (root_lock)
spin_unlock(root_lock);
tcf_kfree_skb_list(to_free);
}
rcu_read_unlock();

8
net/sched/act_ct.c
View File

@ -948,9 +948,9 @@ static int tcf_ct_act_nat(struct sk_buff *skb,
return err & NF_VERDICT_MASK;
if (action & BIT(NF_NAT_MANIP_SRC))
tc_skb_cb(skb)->post_ct_snat = 1;
qdisc_skb_cb(skb)->post_ct_snat = 1;
if (action & BIT(NF_NAT_MANIP_DST))
tc_skb_cb(skb)->post_ct_dnat = 1;
qdisc_skb_cb(skb)->post_ct_dnat = 1;
return err;
#else
@ -986,7 +986,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
tcf_action_update_bstats(&c->common, skb);
if (clear) {
tc_skb_cb(skb)->post_ct = false;
qdisc_skb_cb(skb)->post_ct = false;
ct = nf_ct_get(skb, &ctinfo);
if (ct) {
nf_ct_put(ct);
@ -1097,7 +1097,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
out_push:
skb_push_rcsum(skb, nh_ofs);
tc_skb_cb(skb)->post_ct = true;
qdisc_skb_cb(skb)->post_ct = true;
tc_skb_cb(skb)->zone = p->zone;
out_clear:
if (defrag)

6
net/sched/cls_api.c
View File

@ -1872,9 +1872,9 @@ int tcf_classify(struct sk_buff *skb,
}
ext->chain = last_executed_chain;
ext->mru = cb->mru;
ext->post_ct = cb->post_ct;
ext->post_ct_snat = cb->post_ct_snat;
ext->post_ct_dnat = cb->post_ct_dnat;
ext->post_ct = qdisc_skb_cb(skb)->post_ct;
ext->post_ct_snat = qdisc_skb_cb(skb)->post_ct_snat;
ext->post_ct_dnat = qdisc_skb_cb(skb)->post_ct_dnat;
ext->zone = cb->zone;
}
}

2
net/sched/cls_flower.c
View File

@ -326,7 +326,7 @@ TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb,
struct tcf_result *res)
{
struct cls_fl_head *head = rcu_dereference_bh(tp->root);
bool post_ct = tc_skb_cb(skb)->post_ct;
bool post_ct = qdisc_skb_cb(skb)->post_ct;
u16 zone = tc_skb_cb(skb)->zone;
struct fl_flow_key skb_key;
struct fl_flow_mask *mask;

19
net/sched/sch_cake.c
View File

@ -1398,15 +1398,15 @@ static u32 cake_overhead(struct cake_sched_data *q, const struct sk_buff *skb)
const struct skb_shared_info *shinfo = skb_shinfo(skb);
unsigned int hdr_len, last_len = 0;
u32 off = skb_network_offset(skb);
u16 segs = qdisc_pkt_segs(skb);
u32 len = qdisc_pkt_len(skb);
u16 segs = 1;
q->avg_netoff = cake_ewma(q->avg_netoff, off << 16, 8);
if (!shinfo->gso_size)
if (segs == 1)
return cake_calc_overhead(q, len, off);
/* borrowed from qdisc_pkt_len_init() */
/* borrowed from qdisc_pkt_len_segs_init() */
if (!skb->encapsulation)
hdr_len = skb_transport_offset(skb);
else
@ -1430,12 +1430,6 @@ static u32 cake_overhead(struct cake_sched_data *q, const struct sk_buff *skb)
hdr_len += sizeof(struct udphdr);
}
if (unlikely(shinfo->gso_type & SKB_GSO_DODGY))
segs = DIV_ROUND_UP(skb->len - hdr_len,
shinfo->gso_size);
else
segs = shinfo->gso_segs;
len = shinfo->gso_size + hdr_len;
last_len = skb->len - shinfo->gso_size * (segs - 1);
@ -1788,7 +1782,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (unlikely(len > b->max_skblen))
b->max_skblen = len;
if (skb_is_gso(skb) && q->rate_flags & CAKE_FLAG_SPLIT_GSO) {
if (qdisc_pkt_segs(skb) > 1 && q->rate_flags & CAKE_FLAG_SPLIT_GSO) {
struct sk_buff *segs, *nskb;
netdev_features_t features = netif_skb_features(skb);
unsigned int slen = 0, numsegs = 0;
@ -1800,6 +1794,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
skb_list_walk_safe(segs, segs, nskb) {
skb_mark_not_on_list(segs);
qdisc_skb_cb(segs)->pkt_len = segs->len;
qdisc_skb_cb(segs)->pkt_segs = 1;
cobalt_set_enqueue_time(segs, now);
get_cobalt_cb(segs)->adjusted_len = cake_overhead(q,
segs);
@ -2188,7 +2183,7 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
b->tin_dropped++;
qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
qdisc_qstats_drop(sch);
kfree_skb_reason(skb, reason);
qdisc_dequeue_drop(sch, skb, reason);
if (q->rate_flags & CAKE_FLAG_INGRESS)
goto retry;
}
@ -2729,6 +2724,8 @@ static int cake_init(struct Qdisc *sch, struct nlattr *opt,
int i, j, err;
sch->limit = 10240;
sch->flags |= TCQ_F_DEQUEUE_DROPS;
q->tin_mode = CAKE_DIFFSERV_DIFFSERV3;
q->flow_mode = CAKE_FLOW_TRIPLE;

4
net/sched/sch_codel.c
View File

@ -52,7 +52,7 @@ static void drop_func(struct sk_buff *skb, void *ctx)
{
struct Qdisc *sch = ctx;
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_CONGESTED);
qdisc_dequeue_drop(sch, skb, SKB_DROP_REASON_QDISC_CONGESTED);
qdisc_qstats_drop(sch);
}
@ -182,6 +182,8 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt,
else
sch->flags &= ~TCQ_F_CAN_BYPASS;
sch->flags |= TCQ_F_DEQUEUE_DROPS;
return 0;
}

1
net/sched/sch_dualpi2.c
View File

@ -475,6 +475,7 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
* (3) Enqueue fragment & set ts in dualpi2_enqueue_skb
*/
qdisc_skb_cb(nskb)->pkt_len = nskb->len;
qdisc_skb_cb(nskb)->pkt_segs = 1;
dualpi2_skb_cb(nskb)->classified =
dualpi2_skb_cb(skb)->classified;
dualpi2_skb_cb(nskb)->ect = dualpi2_skb_cb(skb)->ect;

9
net/sched/sch_fq.c
View File

@ -480,7 +480,10 @@ static void fq_erase_head(struct Qdisc *sch, struct fq_flow *flow,
struct sk_buff *skb)
{
if (skb == flow->head) {
flow->head = skb->next;
struct sk_buff *next = skb->next;
prefetch(next);
flow->head = next;
} else {
rb_erase(&skb->rbnode, &flow->t_root);
skb->dev = qdisc_dev(sch);
@ -497,6 +500,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow,
skb_mark_not_on_list(skb);
qdisc_qstats_backlog_dec(sch, skb);
sch->q.qlen--;
qdisc_bstats_update(sch, skb);
}
static void flow_queue_add(struct fq_flow *flow, struct sk_buff *skb)
@ -711,6 +715,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
goto begin;
}
prefetch(&skb->end);
fq_dequeue_skb(sch, f, skb);
if ((s64)(now - time_next_packet - q->ce_threshold) > 0) {
INET_ECN_set_ce(skb);
q->stat_ce_mark++;
@ -718,7 +723,6 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
if (--f->qlen == 0)
q->inactive_flows++;
q->band_pkt_count[fq_skb_cb(skb)->band]--;
fq_dequeue_skb(sch, f, skb);
} else {
head->first = f->next;
/* force a pass through old_flows to prevent starvation */
@ -776,7 +780,6 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
f->time_next_packet = now + len;
}
out:
qdisc_bstats_update(sch, skb);
return skb;
}

5
net/sched/sch_fq_codel.c
View File

@ -275,7 +275,7 @@ static void drop_func(struct sk_buff *skb, void *ctx)
{
struct Qdisc *sch = ctx;
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_CONGESTED);
qdisc_dequeue_drop(sch, skb, SKB_DROP_REASON_QDISC_CONGESTED);
qdisc_qstats_drop(sch);
}
@ -519,6 +519,9 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt,
sch->flags |= TCQ_F_CAN_BYPASS;
else
sch->flags &= ~TCQ_F_CAN_BYPASS;
sch->flags |= TCQ_F_DEQUEUE_DROPS;
return 0;
alloc_failure:

1
net/sched/sch_netem.c
View File

@ -429,6 +429,7 @@ static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff *segs;
netdev_features_t features = netif_skb_features(skb);
qdisc_skb_cb(skb)->pkt_segs = 1;
segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
if (IS_ERR_OR_NULL(segs)) {

2
net/sched/sch_qfq.c
View File

@ -1250,7 +1250,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
}
gso_segs = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
gso_segs = qdisc_pkt_segs(skb);
err = qdisc_enqueue(skb, cl->qdisc, to_free);
if (unlikely(err != NET_XMIT_SUCCESS)) {
pr_debug("qfq_enqueue: enqueue failed %d\n", err);

1
net/sched/sch_taprio.c
View File

@ -595,6 +595,7 @@ static int taprio_enqueue_segmented(struct sk_buff *skb, struct Qdisc *sch,
skb_list_walk_safe(segs, segs, nskb) {
skb_mark_not_on_list(segs);
qdisc_skb_cb(segs)->pkt_len = segs->len;
qdisc_skb_cb(segs)->pkt_segs = 1;
slen += segs->len;
/* FIXME: we should be segmenting to a smaller size

1
net/sched/sch_tbf.c
View File

@ -221,6 +221,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch,
skb_mark_not_on_list(segs);
seg_len = segs->len;
qdisc_skb_cb(segs)->pkt_len = seg_len;
qdisc_skb_cb(segs)->pkt_segs = 1;
ret = qdisc_enqueue(segs, q->qdisc, to_free);
if (ret != NET_XMIT_SUCCESS) {
if (net_xmit_drop_count(ret))