diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3cff2174dc03..ac257221c7c5 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2545,6 +2545,11 @@ struct net_device { /* protected by rtnl_lock */ struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; +#ifdef CONFIG_XDP_SOCKETS + /** @xsk_pools: assigned AF_XDP pools, ops protected */ + struct hlist_head xsk_pools; +#endif + u8 dev_addr_shadow[MAX_ADDR_LEN]; netdevice_tracker linkwatch_dev_tracker; netdevice_tracker watchdog_dev_tracker; diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index a7df573784fd..02e89350faf9 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -48,6 +48,8 @@ struct xsk_buff_pool { struct device *dev; struct net_device *netdev; struct list_head xsk_tx_list; + /* Assigned pools, including deferred releases; ops protected. */ + struct hlist_node dev_node; /* Protects modifications to the xsk_tx_list */ spinlock_t xsk_tx_list_lock; refcount_t users; @@ -119,6 +121,7 @@ bool xp_put_pool(struct xsk_buff_pool *pool); void xp_clear_dev(struct xsk_buff_pool *pool); void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); +void xp_clear_dev_all(struct net_device *dev); /* AF_XDP, and XDP core. */ void xp_free(struct xdp_buff_xsk *xskb); diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 33475b180ea6..781ba46eb152 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -2145,6 +2145,11 @@ static int xsk_notifier(struct notifier_block *this, mutex_unlock(&xs->mutex); } mutex_unlock(&net->xdp.lock); + /* + * A released socket is no longer on xdp.list. Its pool can still + * hold a device reference on the frozen release workqueue. + */ + xp_clear_dev_all(dev); break; } return NOTIFY_DONE; diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 9d2d94f1fb75..0ebd4d46f7ab 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -172,7 +172,7 @@ int xp_assign_dev(struct xsk_buff_pool *pool, u32 frame_size; int err = 0; - ASSERT_RTNL(); + netdev_assert_locked_ops_compat(netdev); force_zc = flags & XDP_ZEROCOPY; force_copy = flags & XDP_COPY; @@ -204,6 +204,11 @@ int xp_assign_dev(struct xsk_buff_pool *pool, pool->cached_need_wakeup = XDP_WAKEUP_TX; dev_hold(netdev); + /* + * Socket removal precedes the final pool put. Keep the pool visible to + * NETDEV_UNREGISTER even while release work is waiting for process thaw. + */ + hlist_add_head(&pool->dev_node, &netdev->xsk_pools); if (force_copy) /* For copy-mode, we are done. */ @@ -244,7 +249,6 @@ int xp_assign_dev(struct xsk_buff_pool *pool, bpf.xsk.pool = pool; bpf.xsk.queue_id = queue_id; - netdev_assert_locked_ops_compat(netdev); err = netdev->netdev_ops->ndo_bpf(netdev, &bpf); if (err) goto err_unreg_pool; @@ -265,6 +269,8 @@ int xp_assign_dev(struct xsk_buff_pool *pool, err = 0; /* fallback to copy mode */ if (err) { xsk_clear_pool_at_qid(netdev, queue_id); + hlist_del(&pool->dev_node); + pool->netdev = NULL; dev_put(netdev); } return err; @@ -287,19 +293,39 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, return xp_assign_dev(pool, dev, queue_id, flags); } +static void __xp_clear_dev(struct xsk_buff_pool *pool) +{ + struct net_device *netdev = pool->netdev; + + netdev_assert_locked_ops_compat(netdev); + xp_disable_drv_zc(pool); + xsk_clear_pool_at_qid(netdev, pool->queue_id); + hlist_del(&pool->dev_node); + pool->netdev = NULL; + dev_put(netdev); +} + void xp_clear_dev(struct xsk_buff_pool *pool) { struct net_device *netdev = pool->netdev; - if (!pool->netdev) + if (!netdev) return; netdev_lock_ops(netdev); - xp_disable_drv_zc(pool); - xsk_clear_pool_at_qid(pool->netdev, pool->queue_id); - pool->netdev = NULL; + __xp_clear_dev(pool); netdev_unlock_ops(netdev); - dev_put(netdev); +} + +void xp_clear_dev_all(struct net_device *dev) +{ + struct xsk_buff_pool *pool; + struct hlist_node *next; + + netdev_lock_ops(dev); + hlist_for_each_entry_safe(pool, next, &dev->xsk_pools, dev_node) + __xp_clear_dev(pool); + netdev_unlock_ops(dev); } static void xp_release_deferred(struct work_struct *work) @@ -337,7 +363,11 @@ bool xp_put_pool(struct xsk_buff_pool *pool) if (refcount_dec_and_test(&pool->users)) { INIT_WORK(&pool->work, xp_release_deferred); - schedule_work(&pool->work); + /* + * Teardown calls ndo_bpf(), which may need powered hardware. + * RTNL alone does not exclude the device's system PM callbacks. + */ + queue_work(system_freezable_wq, &pool->work); return true; }