diff --git a/net/core/dev.c b/net/core/dev.c index f32353b0467c..1ad5a47a4409 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -7073,16 +7073,22 @@ static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget) { struct llist_node *free_list; struct sk_buff *skb, *next; + long n = 0; if (llist_empty(&sdn->defer_list)) return; - atomic_long_set(&sdn->defer_count, 0); free_list = llist_del_all(&sdn->defer_list); llist_for_each_entry_safe(skb, next, free_list, ll_node) { prefetch(next); napi_consume_skb(skb, budget); + n++; } + /* Subtract exactly what was drained. Never store an absolute value: + * mixing one with the relative updates in skb_attempt_defer_free() + * lets the counter go negative, and it is read as unsigned there. + */ + atomic_long_sub(n, &sdn->defer_count); } void skb_defer_node_flush(struct skb_defer_node *sdn) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 48f676411724..68192e477ff1 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -7499,8 +7499,14 @@ nodefer: kfree_skb_napi_cache(skb); defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); defer_count = atomic_long_inc_return(&sdn->defer_count); - if (defer_count >= defer_max) + if (defer_count > defer_max) { + /* Undo the reservation. An llist_add() does not release one -- + * the skb stays charged until a flush subtracts the batch that + * contains it -- so a rejected free must undo its own. + */ + atomic_long_dec(&sdn->defer_count); goto nodefer; + } llist_add(&skb->ll_node, &sdn->defer_list); @@ -7509,8 +7515,13 @@ nodefer: kfree_skb_napi_cache(skb); return; } - /* Send an IPI every time queue reaches half capacity. */ - kick = (defer_count - 1) == (defer_max >> 1); + /* Half capacity, plus the skb that fills the queue. Equality alone can + * be consumed early -- if the producer holding the kick value publishes + * first, its purge drains only that skb and the reservations published + * after it match nothing -- which would leave a full queue with nothing + * pending. + */ + kick = (defer_count - 1) == (defer_max >> 1) || defer_count == defer_max; /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU * if we are unlucky enough (this seems very unlikely).