From 212a922bd54507ff18057b5872d552f4ef18ba0e Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 Jun 2026 09:10:10 -0700 Subject: [PATCH 1/5] netconsole: do not schedule skb pool refill from NMI When alloc_skb() fails in find_skb(), the fallback path dequeues an skb from np->skb_pool and unconditionally calls schedule_work() to top the pool back up. schedule_work() ends up taking the workqueue pool locks, which are not NMI-safe. netconsole_write() is registered as the nbcon write_atomic callback and is explicitly marked CON_NBCON_ATOMIC_UNSAFE, meaning it is invoked from emergency/panic contexts including NMIs. If the NMI interrupts a thread already holding the workqueue pool lock, calling schedule_work() self-deadlocks and the panic message that was being printed is lost. Introduce netcons_skb_pop() to fold the pool dequeue and the refill request into a single helper. The helper skips schedule_work() when called from NMI context; the pool is best-effort, so the refill is simply deferred to the next non-NMI find_skb() call that exhausts alloc_skb() and hits the fallback again. This keeps the fast path untouched and the locking rules around the fallback pool documented in one place. Note this only removes the schedule_work() hazard from the NMI path. The allocation itself is still not fully NMI-safe: the alloc_skb(GFP_ATOMIC) attempted first may take slab locks, and the skb_dequeue() fallback takes np->skb_pool.lock, so either can deadlock if the NMI interrupts a holder of those locks. Closing those windows requires an NMI-safe (lockless) skb pool and is left to a follow-up; this patch addresses the schedule_work() deadlock, which is both the most likely and the easiest to trigger. Signed-off-by: Breno Leitao Link: https://patch.msgid.link/20260604-netcons_fix_before_move-v3-1-ab055b3a6aa5@debian.org Signed-off-by: Paolo Abeni --- drivers/net/netconsole.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 8ecc2c71c699..918e4a9f4456 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1654,6 +1654,23 @@ static struct notifier_block netconsole_netdev_notifier = { .notifier_call = netconsole_netdev_event, }; +/* Pop a pre-allocated skb from the pool and request a refill. + * + * The refill is requested via schedule_work(), which takes the workqueue + * pool locks and is therefore not NMI-safe. Skip the refill when called + * from NMI context; the next non-NMI caller will top the pool back up. + */ +static struct sk_buff *netcons_skb_pop(struct netpoll *np) +{ + struct sk_buff *skb; + + skb = skb_dequeue(&np->skb_pool); + if (!in_nmi()) + schedule_work(&np->refill_wq); + + return skb; +} + static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve) { int count = 0; @@ -1663,10 +1680,8 @@ static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve) repeat: skb = alloc_skb(len, GFP_ATOMIC); - if (!skb) { - skb = skb_dequeue(&np->skb_pool); - schedule_work(&np->refill_wq); - } + if (!skb) + skb = netcons_skb_pop(np); if (!skb) { if (++count < 10) { From 6c537b845c99e32312a1bd84d4c95cdb26efb577 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 Jun 2026 09:10:11 -0700 Subject: [PATCH 2/5] netconsole: do not dequeue pooled skbs that cannot satisfy len find_skb() falls back to np->skb_pool when the GFP_ATOMIC alloc_skb() fails. The pool is refilled by refill_skbs(), which always allocates buffers of MAX_SKB_SIZE (ethhdr + iphdr + udphdr + MAX_UDP_CHUNK == 1502 bytes). netconsole, however, computes the requested length dynamically as total_len + np->dev->needed_tailroom If the egress device declares a non-zero needed_tailroom (e.g. some tunnel or hardware accelerator devices), the required length can exceed MAX_SKB_SIZE. The pooled skb is then handed back to the caller, which immediately performs skb_put(skb, len), trips the tail > end check, and triggers skb_over_panic(). Leave the normal alloc_skb(len, GFP_ATOMIC) path untouched -- the slab allocator can still satisfy oversized requests when memory is available, so senders to devices with non-zero needed_tailroom keep working in the common case. Only the pool fallback is gated: when alloc_skb() failed and len exceeds the pool buffer size, skip the skb_dequeue() instead of burning a pre-allocated skb on a request that would later trip skb_over_panic(). Reserving pool entries for requests they can actually satisfy also keeps the panic path, which depends on the pool being primed, intact. When that drop happens, emit a rate-limited net_warn() so the user notices that netconsole is unable to push messages on the egress device. The warn is skipped under in_nmi() for the same reason schedule_work() is: printk machinery taken by net_warn_ratelimited() is not NMI-safe and would risk recursing into the same nbcon console we are servicing. MAX_SKB_SIZE / MAX_UDP_CHUNK were private to net/core/netpoll.c. Move them to include/linux/netpoll.h so netconsole can reference the same definition that refill_skbs() uses, keeping the two in sync by construction. The header now pulls in and explicitly so MAX_SKB_SIZE remains self-contained for any future user. Signed-off-by: Breno Leitao Link: https://patch.msgid.link/20260604-netcons_fix_before_move-v3-2-ab055b3a6aa5@debian.org Signed-off-by: Paolo Abeni --- drivers/net/netconsole.c | 22 ++++++++++++++++++++-- include/linux/netpoll.h | 16 ++++++++++++++++ net/core/netpoll.c | 7 ------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 918e4a9f4456..58250e648f8b 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1655,15 +1655,33 @@ static struct notifier_block netconsole_netdev_notifier = { }; /* Pop a pre-allocated skb from the pool and request a refill. + * + * The pool is refilled with MAX_SKB_SIZE buffers, so a pooled skb cannot + * satisfy a larger request. Return NULL in that case rather than handing + * back a too-small skb that would later trip skb_over_panic() in skb_put(); + * the caller still polls and retries, and alloc_skb() itself can satisfy the + * oversized request once memory frees up. * * The refill is requested via schedule_work(), which takes the workqueue * pool locks and is therefore not NMI-safe. Skip the refill when called * from NMI context; the next non-NMI caller will top the pool back up. */ -static struct sk_buff *netcons_skb_pop(struct netpoll *np) +static struct sk_buff *netcons_skb_pop(struct netpoll *np, int len) { struct sk_buff *skb; + if (len > MAX_SKB_SIZE) { + /* net_warn_ratelimited() pulls in printk machinery that is not + * NMI-safe and could recurse into the nbcon console we are + * servicing, so only warn outside NMI. + */ + if (!in_nmi()) + net_warn_ratelimited("netconsole: dropping message, requested skb len %d exceeds pool buffer size %zu on %s\n", + len, (size_t)MAX_SKB_SIZE, + np->dev->name); + return NULL; + } + skb = skb_dequeue(&np->skb_pool); if (!in_nmi()) schedule_work(&np->refill_wq); @@ -1681,7 +1699,7 @@ static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve) skb = alloc_skb(len, GFP_ATOMIC); if (!skb) - skb = netcons_skb_pop(np); + skb = netcons_skb_pop(np, len); if (!skb) { if (++count < 10) { diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index e4b8f1f91e54..88f7daa8560e 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -13,12 +13,28 @@ #include #include #include +#include +#include union inet_addr { __be32 ip; struct in6_addr in6; }; +/* + * Maximum payload netpoll's preallocated skb pool can carry. Keep this in + * sync with the buffer size used by refill_skbs() in net/core/netpoll.c; + * callers (e.g. netconsole) use it to detect requests the pool can never + * satisfy and avoid dequeuing a pooled skb that would later trip + * skb_over_panic() in skb_put(). + */ +#define MAX_UDP_CHUNK 1460 +#define MAX_SKB_SIZE \ + (sizeof(struct ethhdr) + \ + sizeof(struct iphdr) + \ + sizeof(struct udphdr) + \ + MAX_UDP_CHUNK) + struct netpoll { struct net_device *dev; netdevice_tracker dev_tracker; diff --git a/net/core/netpoll.c b/net/core/netpoll.c index b3fe59445f2d..229dde818ab3 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -41,16 +41,9 @@ * message gets out even in extreme OOM situations. */ -#define MAX_UDP_CHUNK 1460 #define MAX_SKBS 32 #define USEC_PER_POLL 50 -#define MAX_SKB_SIZE \ - (sizeof(struct ethhdr) + \ - sizeof(struct iphdr) + \ - sizeof(struct udphdr) + \ - MAX_UDP_CHUNK) - static unsigned int carrier_timeout = 4; module_param(carrier_timeout, uint, 0644); From 91aeb87f052367a5a2743cc93777dfb4386f2f14 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 Jun 2026 09:10:12 -0700 Subject: [PATCH 3/5] netconsole: take target_cleanup_list_lock in drop_netconsole_target() drop_netconsole_target() unlinks the target while only holding target_list_lock. However, when the underlying interface has been unregistered, netconsole_netdev_event() moves the target from target_list to target_cleanup_list, and netconsole_process_cleanups_core() walks that list under target_cleanup_list_lock only. If a user removes the configfs target at the same time the cleanup worker is iterating target_cleanup_list, list_del() can corrupt the list because the two paths take disjoint locks while operating on the same list node. Acquire target_cleanup_list_lock around the list_del() so the unlink is serialised against netconsole_process_cleanups_core() regardless of which list the target currently belongs to. The state transition that downgrades STATE_DEACTIVATED to STATE_DISABLED is left intact and is performed under the same combined locking, preserving the existing ordering with resume_target(). Signed-off-by: Breno Leitao Link: https://patch.msgid.link/20260604-netcons_fix_before_move-v3-3-ab055b3a6aa5@debian.org Signed-off-by: Paolo Abeni --- drivers/net/netconsole.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 58250e648f8b..d8be2fef3826 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1452,6 +1452,7 @@ static void drop_netconsole_target(struct config_group *group, dynamic_netconsole_mutex_lock(); + mutex_lock(&target_cleanup_list_lock); spin_lock_irqsave(&target_list_lock, flags); /* Disable deactivated target to prevent races between resume attempt * and target removal. @@ -1460,6 +1461,7 @@ static void drop_netconsole_target(struct config_group *group, nt->state = STATE_DISABLED; list_del(&nt->list); spin_unlock_irqrestore(&target_list_lock, flags); + mutex_unlock(&target_cleanup_list_lock); dynamic_netconsole_mutex_unlock(); From 4cfcd6acc295c6e7cf82ad4f19b1f86f08f3bb78 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 Jun 2026 09:10:13 -0700 Subject: [PATCH 4/5] netconsole: clean up deactivated targets dropped before the cleanup worker drop_netconsole_target() downgrades a STATE_DEACTIVATED target to STATE_DISABLED and then only calls netpoll_cleanup() when the target is STATE_ENABLED. A target becomes STATE_DEACTIVATED when its underlying interface is unregistered: netconsole_netdev_event() moves it to target_cleanup_list, and netconsole_process_cleanups_core() is expected to run do_netpoll_cleanup() on it. Now that drop_netconsole_target() takes target_cleanup_list_lock around the unlink, a configfs removal racing with NETDEV_UNREGISTER can pull the target off target_cleanup_list before the cleanup worker processes it. The notifier drops the lock before calling netconsole_process_cleanups_core(), so the worker then iterates a list that no longer contains the target and never runs do_netpoll_cleanup() on it. Because drop_netconsole_target() has already rewritten the state to STATE_DISABLED, its own STATE_ENABLED check is false and netpoll_cleanup() is skipped too. The net_device reference taken by netpoll_setup() is then leaked and unregister_netdevice() hangs forever in netdev_wait_allrefs(). Capture whether the target still owns a netpoll before the state is downgraded and clean it up for both STATE_ENABLED and STATE_DEACTIVATED targets. netpoll_cleanup() is idempotent -- it skips when np->dev is already NULL -- so it is safe even when the cleanup worker won the race and already tore the netpoll down. Signed-off-by: Breno Leitao Link: https://patch.msgid.link/20260604-netcons_fix_before_move-v3-4-ab055b3a6aa5@debian.org Signed-off-by: Paolo Abeni --- drivers/net/netconsole.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index d8be2fef3826..80c5393ffa1c 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1449,11 +1449,21 @@ static void drop_netconsole_target(struct config_group *group, { struct netconsole_target *nt = to_target(item); unsigned long flags; + bool needs_cleanup; dynamic_netconsole_mutex_lock(); mutex_lock(&target_cleanup_list_lock); spin_lock_irqsave(&target_list_lock, flags); + /* A STATE_DEACTIVATED target may have been moved to + * target_cleanup_list by netconsole_netdev_event() but not yet + * processed by netconsole_process_cleanups_core(). Unlinking it below + * hides it from the cleanup worker, so this path has to clean it up + * itself. Record that the target still owns a netpoll before the + * state is downgraded. + */ + needs_cleanup = nt->state == STATE_ENABLED || + nt->state == STATE_DEACTIVATED; /* Disable deactivated target to prevent races between resume attempt * and target removal. */ @@ -1475,8 +1485,10 @@ static void drop_netconsole_target(struct config_group *group, /* * The target may have never been enabled, or was manually disabled * before being removed so netpoll may have already been cleaned up. + * netpoll_cleanup() is idempotent (it skips when np->dev is NULL), so + * it is safe even if the cleanup worker already tore the netpoll down. */ - if (nt->state == STATE_ENABLED) + if (needs_cleanup) netpoll_cleanup(&nt->np); config_item_put(&nt->group.cg_item); From 0360976d7ed5c69484d873afa34a22db4d04996f Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 Jun 2026 09:10:14 -0700 Subject: [PATCH 5/5] netconsole: close netdevice unregister window during target resume process_resume_target() removes the target from target_list before calling resume_target() so that netpoll_setup() can run with interrupts enabled, then re-adds it once setup completes. netpoll_setup() acquires a net_device reference (netdev_hold()) and releases the RTNL before returning. While the target is off target_list and the RTNL is not held, netconsole_netdev_event() cannot find it. If the egress device is unregistered in that window, the NETDEV_UNREGISTER notifier walks target_list, misses the resuming target, and never tears it down. The target is then re-added in STATE_ENABLED still holding a reference to the now-unregistered device, leaking it and hanging unregister_netdevice() in netdev_wait_allrefs(). Re-check under RTNL before re-publishing the target: if the device left NETREG_REGISTERED while we were off the list, run do_netpoll_cleanup() and mark the target disabled. Taking the RTNL across the check and the list_add() serialises against the NETDEV_UNREGISTER notifier, which also runs under RTNL, so the device is either still registered (and the notifier will find the re-added target later) or already unregistering (and we drop the reference here). netdev_wait_allrefs() runs from netdev_run_todo() outside the RTNL, so dropping the reference here cannot deadlock against the pending unregister. Signed-off-by: Breno Leitao Link: https://patch.msgid.link/20260604-netcons_fix_before_move-v3-5-ab055b3a6aa5@debian.org Signed-off-by: Paolo Abeni --- drivers/net/netconsole.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 80c5393ffa1c..606e265cdfd7 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -335,6 +335,24 @@ static void process_resume_target(struct work_struct *work) resume_target(nt); + /* netpoll_setup() took a net_device reference and dropped the RTNL + * before returning, all while this target was off target_list and + * thus invisible to netconsole_netdev_event(). If the device was + * unregistered in that window the NETDEV_UNREGISTER notifier could not + * tear this target down, which would leak the reference and hang + * unregister_netdevice(). Re-check under the RTNL before re-publishing: + * taking it across the check and the list_add() serialises against the + * notifier (which also runs under the RTNL), so the device is either + * still registered (the notifier will find the re-added target) or + * already unregistering (we drop the reference here). + */ + rtnl_lock(); + if (nt->state == STATE_ENABLED && nt->np.dev && + nt->np.dev->reg_state != NETREG_REGISTERED) { + do_netpoll_cleanup(&nt->np); + nt->state = STATE_DISABLED; + } + /* At this point the target is either enabled or disabled and * was cleaned up before getting deactivated. Either way, add it * back to target list. @@ -342,6 +360,7 @@ static void process_resume_target(struct work_struct *work) spin_lock_irqsave(&target_list_lock, flags); list_add(&nt->list, &target_list); spin_unlock_irqrestore(&target_list_lock, flags); + rtnl_unlock(); out_unlock: dynamic_netconsole_mutex_unlock();