From 1cf2f913b3dd934b339e0d50d3447a9b1c07b83f Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 25 Sep 2026 13:15:20 -0700 Subject: [PATCH 1/3] net: netmem: move to index based freelist Move devmem's freelist to an index-based approach similar to io_uring. This halves the memory requirements for the freelist. Reviewed-by: Mina Almasry Signed-off-by: Stanislav Fomichev Link: https://patch.msgid.link/20260925201522.254717-2-sdf@fomichev.me Signed-off-by: Paolo Abeni --- net/core/devmem.c | 50 ++++++++++++++++++++++++++++++++++++----------- net/core/devmem.h | 4 ++-- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/net/core/devmem.c b/net/core/devmem.c index a9d86b5a5588..704d02bd8bc8 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -29,6 +29,29 @@ static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1); static const struct memory_provider_ops dmabuf_devmem_ops; +static struct net_iov *net_iov_free_pop(struct net_iov_area *area, + u32 *freelist, u32 *free_count) +{ + u32 idx; + + if (unlikely(!*free_count)) + return NULL; + + idx = freelist[--(*free_count)]; + return &area->niovs[idx]; +} + +static void net_iov_free_push(struct net_iov_area *area, + u32 *freelist, u32 *free_count, + struct net_iov *niov) +{ + if (WARN_ON_ONCE(net_iov_owner(niov) != area || + *free_count >= area->num_niovs)) + return; + + freelist[(*free_count)++] = net_iov_idx(niov); +} + static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) { struct net_devmem_dmabuf_binding *binding = @@ -44,7 +67,7 @@ void __net_devmem_dmabuf_binding_free(struct work_struct *wq) if (binding->freelist) WARN(binding->free_count != binding->area.num_niovs, - "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%zu", + "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%u", binding->area.num_niovs, binding->free_count); kvfree(binding->area.niovs); @@ -63,14 +86,15 @@ static unsigned int net_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding, netmem_ref *netmems, unsigned int count) { + struct net_iov *niov; unsigned int i; spin_lock_bh(&binding->freelist_lock); - count = min_t(size_t, count, binding->free_count); + count = min(count, binding->free_count); for (i = 0; i < count; i++) { - struct net_iov *niov = binding->freelist[--binding->free_count]; - + niov = net_iov_free_pop(&binding->area, binding->freelist, + &binding->free_count); netmems[i] = net_iov_to_netmem(niov); } @@ -84,12 +108,8 @@ void net_devmem_free_dmabuf(struct net_iov *niov) struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); spin_lock_bh(&binding->freelist_lock); - if (WARN_ON_ONCE(binding->free_count >= binding->area.num_niovs)) { - spin_unlock_bh(&binding->freelist_lock); - return; - } - - binding->freelist[binding->free_count++] = niov; + net_iov_free_push(&binding->area, binding->freelist, + &binding->free_count, niov); spin_unlock_bh(&binding->freelist_lock); } @@ -228,6 +248,12 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, goto err_unmap; } + if ((dmabuf->size >> niov_shift) > U32_MAX) { + err = -E2BIG; + NL_SET_ERR_MSG(extack, "dmabuf contains too many net_iovs"); + goto err_unmap; + } + binding->area.base_virtual = 0; binding->area.num_niovs = dmabuf->size >> niov_shift; if (direction == DMA_TO_DEVICE) { @@ -279,7 +305,9 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, if (direction == DMA_TO_DEVICE) binding->tx_vec[niov_idx] = niov; else - binding->freelist[binding->free_count++] = niov; + net_iov_free_push(&binding->area, + binding->freelist, + &binding->free_count, niov); dma_addr += niov_size; } } diff --git a/net/core/devmem.h b/net/core/devmem.h index f71d9a2bcb3d..28d6fa7bcb57 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -58,8 +58,8 @@ struct net_devmem_dmabuf_binding { struct xarray bound_rxqs; spinlock_t freelist_lock ____cacheline_aligned_in_smp; - size_t free_count; - struct net_iov **freelist; + u32 free_count; + u32 *freelist; /* ID of this binding. Globally unique to all bindings currently * active. From 5bdeb89ec5b07c751c490af7cac9b1e20a1426ac Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 25 Sep 2026 13:15:21 -0700 Subject: [PATCH 2/3] net: devmem: use memory provider helpers for net_iovs Use public memory-provider helpers for DMA address storage and page-pool association instead of open-coding page_pool state and accounting. Zero the net_iov array because net_mp_niov_set_page_pool() ORs PP_SIGNATURE into pp_magic. Reject DMA addresses that cannot be represented in net_iov metadata. Reviewed-by: Mina Almasry Signed-off-by: Stanislav Fomichev Link: https://patch.msgid.link/20260925201522.254717-3-sdf@fomichev.me Signed-off-by: Paolo Abeni --- net/core/devmem.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/net/core/devmem.c b/net/core/devmem.c index 704d02bd8bc8..71c83730d73d 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -17,11 +17,9 @@ #include #include #include -#include #include "devmem.h" #include "mp_dmabuf_devmem.h" -#include "page_pool_priv.h" /* Device memory support */ @@ -273,7 +271,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, goto err_unmap; } } - binding->area.niovs = kvmalloc_objs(*binding->area.niovs, + binding->area.niovs = kvzalloc_objs(*binding->area.niovs, binding->area.num_niovs); if (!binding->area.niovs) { err = -ENOMEM; @@ -300,8 +298,12 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, for (i = 0; i < nr_niovs; i++, niov_idx++) { niov = &binding->area.niovs[niov_idx]; net_iov_init(niov, &binding->area, NET_IOV_DMABUF); - page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), - dma_addr); + if (net_mp_niov_set_dma_addr(niov, dma_addr)) { + err = -EFAULT; + NL_SET_ERR_MSG(extack, + "dmabuf DMA address cannot be represented"); + goto err_free_niovs; + } if (direction == DMA_TO_DEVICE) binding->tx_vec[niov_idx] = niov; else @@ -467,19 +469,9 @@ netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp) if (unlikely(!allocated)) return 0; - for (i = 0; i < allocated; i++) { - struct net_iov *niov = netmem_to_net_iov(netmems[i]); - - niov->desc.pp_magic = 0; - niov->desc.pp = NULL; - atomic_long_set(&niov->desc.pp_ref_count, 0); - - page_pool_set_pp_info(pool, netmems[i]); - - pool->pages_state_hold_cnt++; - trace_page_pool_state_hold(pool, netmems[i], - pool->pages_state_hold_cnt); - } + for (i = 0; i < allocated; i++) + net_mp_niov_set_page_pool(pool, + netmem_to_net_iov(netmems[i])); /* Return the last one, the rest stay in the page_pool cache. */ allocated--; @@ -504,8 +496,7 @@ bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem) if (WARN_ON_ONCE(refcount != 1)) return false; - page_pool_clear_pp_info(netmem); - + net_mp_niov_clear_page_pool(netmem_to_net_iov(netmem)); net_devmem_free_dmabuf(netmem_to_net_iov(netmem)); /* We don't want the page pool put_page()ing our net_iovs. */ From 5c729e6c71f1be48b5afb99326a6b687e3600c67 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 25 Sep 2026 13:15:22 -0700 Subject: [PATCH 3/3] net: devmem: decode DMA addresses for TX On 32-bit architectures where dma_addr_t is wider than unsigned long, page_pool_set_dma_addr_netmem() stores page-aligned DMA addresses shifted by PAGE_SHIFT. The net_iov branch of __skb_frag_dma_map() adds byte offsets to the encoded value, so the NIC is programmed with an invalid DMA address. This can trigger an IOMMU fault or DMA from unintended memory. Consolidate DMA address encoding, decoding, and representability checks in netmem helpers. Use the common decoder from the page pool and net_iov TX paths so both interpret stored addresses consistently. Reviewed-by: Mina Almasry Signed-off-by: Stanislav Fomichev Link: https://patch.msgid.link/20260925201522.254717-4-sdf@fomichev.me Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 7 +++++-- include/net/netmem.h | 27 +++++++++++++++++++++++++++ include/net/page_pool/helpers.h | 10 +--------- net/core/page_pool_priv.h | 14 ++------------ 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 84308498a3a8..38b7d7389452 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3816,9 +3816,12 @@ static inline dma_addr_t __skb_frag_dma_map(struct device *dev, size_t offset, size_t size, enum dma_data_direction dir) { + dma_addr_t addr; + if (skb_frag_is_net_iov(frag)) { - return netmem_to_net_iov(frag->netmem)->desc.dma_addr + - offset + frag->offset; + addr = netmem_dma_addr_decode( + netmem_get_dma_addr(frag->netmem)); + return addr + offset + frag->offset; } return dma_map_page(dev, skb_frag_page(frag), skb_frag_off(frag) + offset, size, dir); diff --git a/include/net/netmem.h b/include/net/netmem.h index bccacd21b6c3..cc97611632dc 100644 --- a/include/net/netmem.h +++ b/include/net/netmem.h @@ -358,6 +358,33 @@ static inline bool netmem_is_pfmemalloc(netmem_ref netmem) return page_is_pfmemalloc(netmem_to_page(netmem)); } +#define NETMEM_32BIT_ARCH_WITH_64BIT_DMA \ + (sizeof(dma_addr_t) > sizeof(unsigned long)) + +static inline unsigned long netmem_dma_addr_encode(dma_addr_t addr) +{ + if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) + addr >>= PAGE_SHIFT; + + return addr; +} + +static inline dma_addr_t netmem_dma_addr_decode(unsigned long addr) +{ + if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) + return (dma_addr_t)addr << PAGE_SHIFT; + + return addr; +} + +static inline bool netmem_dma_addr_fits(dma_addr_t addr) +{ + /* We assume page alignment to shave off bottom bits, + * if this "compression" doesn't work we need to drop. + */ + return addr == netmem_dma_addr_decode(netmem_dma_addr_encode(addr)); +} + static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) { return netmem_to_nmdesc(netmem)->dma_addr; diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index 87a4e13886e1..cd021832c3fa 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -408,9 +408,6 @@ static inline void page_pool_recycle_direct_netmem(struct page_pool *pool, page_pool_put_full_netmem(pool, netmem, true); } -#define PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA \ - (sizeof(dma_addr_t) > sizeof(unsigned long)) - /** * page_pool_free_va() - free a va into the page_pool * @pool: pool from which va was allocated @@ -427,12 +424,7 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem) { - dma_addr_t ret = netmem_get_dma_addr(netmem); - - if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) - ret <<= PAGE_SHIFT; - - return ret; + return netmem_dma_addr_decode(netmem_get_dma_addr(netmem)); } /** diff --git a/net/core/page_pool_priv.h b/net/core/page_pool_priv.h index 2fb06d5f6d55..430b97cd88da 100644 --- a/net/core/page_pool_priv.h +++ b/net/core/page_pool_priv.h @@ -18,18 +18,8 @@ void page_pool_unlist(struct page_pool *pool); static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem, dma_addr_t addr) { - if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) { - netmem_set_dma_addr(netmem, addr >> PAGE_SHIFT); - - /* We assume page alignment to shave off bottom bits, - * if this "compression" doesn't work we need to drop. - */ - return addr != (dma_addr_t)netmem_get_dma_addr(netmem) - << PAGE_SHIFT; - } - - netmem_set_dma_addr(netmem, addr); - return false; + netmem_set_dma_addr(netmem, netmem_dma_addr_encode(addr)); + return !netmem_dma_addr_fits(addr); } static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)