github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-10-04 18:29:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

net: flush skb_defer_nodes in dev_cpu_dead()

Browse Source 
When a CPU goes offline, dev_cpu_dead() drains its softnet queues
(completion_queue, output_queue, poll_list, process_queue, and
input_pkt_queue), but leaves net_hotdata.skb_defer_nodes untouched.

If oldcpu goes offline while holding pending skbs in its
skb_defer_nodes lists (e.g. below the sysctl_skb_defer_max >> 1 IPI
threshold, or if the IPI races with CPU teardown), those skbs remain
stranded until oldcpu is brought back online. If any of these skbs
hold page_pool fragments, page_pool_destroy() will stall indefinitely
waiting for inflight pages to be returned when a netdev or driver is
torn down while oldcpu is offline.

Additionally, if smp_call_function_single_async() fails in
kick_defer_list_purge() because the target CPU went offline, reset
defer_ipi_scheduled to 0 so future IPI kicks are not blocked when the
CPU comes back online.

Also, if oldcpu was the last online CPU on its NUMA node, drain that
node's slot across all CPUs so no skbs deferred from that node remain
stranded on idle remote CPUs (or if the node itself is subsequently
offlined).

Finally, in skb_attempt_defer_free(), re-check cpu_online(cpu) and
whether the caller migrated CPUs after llist_add(), flushing the node
list if so, to close the preemption TOCTOU race against CPU/node
teardown.

Fixes: 68822bdf76 ("net: generalize skb freeing deferral to per-cpu lists")
Fixes: 5628f3fe3b ("net: add NUMA awareness to skb_attempt_defer_free()")
Closes: https://lore.kernel.org/netdev/20260916003430.3612956-1-kris.pan@intel.com/
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Kris Pan <kris.pan@intel.com>
Link: https://patch.msgid.link/20260923130318.607255-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet 2026-09-23 13:03:18 +00:00 committed by Jakub Kicinski
parent 8db67bb6a1
commit 06e3f54e8b
3 changed files with 46 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
net/core/dev.c
View File

@ -5376,7 +5376,8 @@ void kick_defer_list_purge(unsigned int cpu)
backlog_unlock_irq_restore(sd, flags);
} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
smp_call_function_single_async(cpu, &sd->defer_csd);
if (smp_call_function_single_async(cpu, &sd->defer_csd))
WRITE_ONCE(sd->defer_ipi_scheduled, 0);
}
}
@ -6900,25 +6901,35 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
}
EXPORT_SYMBOL(napi_complete_done);
static void skb_defer_free_flush(void)
static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget)
{
struct llist_node *free_list;
struct sk_buff *skb, *next;
if (llist_empty(&sdn->defer_list))
return;
atomic_long_set(&sdn->defer_count, 0);
free_list = llist_del_all(&sdn->defer_list);
llist_for_each_entry_safe(skb, next, free_list, ll_node) {
prefetch(next);
napi_consume_skb(skb, budget);
}
}
void skb_defer_node_flush(struct skb_defer_node *sdn)
{
__skb_defer_free_flush(sdn, 0);
}
static void skb_defer_free_flush(void)
{
struct skb_defer_node *sdn;
int node;
for_each_node(node) {
sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node;
if (llist_empty(&sdn->defer_list))
continue;
atomic_long_set(&sdn->defer_count, 0);
free_list = llist_del_all(&sdn->defer_list);
llist_for_each_entry_safe(skb, next, free_list, ll_node) {
prefetch(next);
napi_consume_skb(skb, 1);
}
__skb_defer_free_flush(sdn, 1);
}
}
@ -12897,6 +12908,7 @@ static int dev_cpu_dead(unsigned int oldcpu)
struct sk_buff **list_skb;
struct sk_buff *skb;
unsigned int cpu;
int node;
struct softnet_data *sd, *oldsd, *remsd = NULL;
local_irq_disable();
@ -12957,6 +12969,17 @@ static int dev_cpu_dead(unsigned int oldcpu)
rps_input_queue_head_incr(oldsd);
}
for_each_node(node)
skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
oldcpu) + node);
node = cpu_to_node(oldcpu);
if (node_possible(node) &&
!cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) {
for_each_possible_cpu(cpu)
skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
cpu) + node);
}
return 0;
}

2
net/core/dev.h
View File

@ -399,6 +399,8 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi)
WARN_ON(READ_ONCE(napi->list_owner) != -1);
}
struct skb_defer_node;
void skb_defer_node_flush(struct skb_defer_node *sdn);
void kick_defer_list_purge(unsigned int cpu);
int dev_set_hwtstamp_phylib(struct net_device *dev,

12
net/core/skbuff.c
View File

@ -7360,8 +7360,8 @@ void skb_attempt_defer_free(struct sk_buff *skb)
struct skb_defer_node *sdn;
unsigned long defer_count;
unsigned int defer_max;
int cpu, my_cpu;
bool kick;
int cpu;
if (static_branch_unlikely(&skb_defer_disable_key))
goto nodefer;
@ -7371,7 +7371,8 @@ void skb_attempt_defer_free(struct sk_buff *skb)
goto nodefer;
cpu = skb->alloc_cpu;
if (cpu == raw_smp_processor_id() ||
my_cpu = raw_smp_processor_id();
if (cpu == my_cpu ||
WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
!cpu_online(cpu)) {
nodefer: kfree_skb_napi_cache(skb);
@ -7382,7 +7383,7 @@ nodefer: kfree_skb_napi_cache(skb);
DEBUG_NET_WARN_ON_ONCE(skb->destructor);
DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb));
sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + numa_node_id();
sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu);
defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
defer_count = atomic_long_inc_return(&sdn->defer_count);
@ -7392,6 +7393,11 @@ nodefer: kfree_skb_napi_cache(skb);
llist_add(&skb->ll_node, &sdn->defer_list);
if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) {
skb_defer_node_flush(sdn);
return;
}
/* Send an IPI every time queue reaches half capacity. */
kick = (defer_count - 1) == (defer_max >> 1);