github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-12 16:18:45 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

page_pool: make page_pool_put_page_bulk() handle array of netmems

Browse Source 
Currently, page_pool_put_page_bulk() indeed takes an array of pointers
to the data, not pages, despite the name. As one side effect, when
you're freeing frags from &skb_shared_info, xdp_return_frame_bulk()
converts page pointers to virtual addresses and then
page_pool_put_page_bulk() converts them back. Moreover, data pointers
assume every frag is placed in the host memory, making this function
non-universal.
Make page_pool_put_page_bulk() handle array of netmems. Pass frag
netmems directly and use virt_to_netmem() when freeing xdpf->data,
so that the PP core will then get the compound netmem and take care
of the rest.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20241203173733.3181246-9-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Alexander Lobakin 2024-12-03 18:37:31 +01:00 committed by Jakub Kicinski
parent 9bd9f72a74
commit 024bfd2e9d
4 changed files with 23 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/net/page_pool/types.h
View File

@ -259,8 +259,8 @@ void page_pool_disable_direct_recycling(struct page_pool *pool);
void page_pool_destroy(struct page_pool *pool);
void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
const struct xdp_mem_info *mem);
void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count);
void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data,
u32 count);
#else
static inline void page_pool_destroy(struct page_pool *pool)
{
@ -272,8 +272,8 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool,
{
}
static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
static inline void page_pool_put_netmem_bulk(struct page_pool *pool,
netmem_ref *data, u32 count)
{
}
#endif

2
include/net/xdp.h
View File

@ -194,7 +194,7 @@ xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame)
struct xdp_frame_bulk {
int count;
void *xa;
void *q[XDP_BULK_QUEUE_SIZE];
netmem_ref q[XDP_BULK_QUEUE_SIZE];
};
static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq)

30
net/core/page_pool.c
View File

@ -840,22 +840,22 @@ void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
EXPORT_SYMBOL(page_pool_put_unrefed_page);
/**
* page_pool_put_page_bulk() - release references on multiple pages
* page_pool_put_netmem_bulk() - release references on multiple netmems
* @pool: pool from which pages were allocated
* @data: array holding page pointers
* @count: number of pages in @data
* @data: array holding netmem references
* @count: number of entries in @data
*
* Tries to refill a number of pages into the ptr_ring cache holding ptr_ring
* producer lock. If the ptr_ring is full, page_pool_put_page_bulk()
* will release leftover pages to the page allocator.
* page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx
* Tries to refill a number of netmems into the ptr_ring cache holding ptr_ring
* producer lock. If the ptr_ring is full, page_pool_put_netmem_bulk()
* will release leftover netmems to the memory provider.
* page_pool_put_netmem_bulk() is suitable to be run inside the driver NAPI tx
* completion loop for the XDP_REDIRECT use case.
*
* Please note the caller must not use data area after running
* page_pool_put_page_bulk(), as this function overwrites it.
* page_pool_put_netmem_bulk(), as this function overwrites it.
*/
void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data,
u32 count)
{
int i, bulk_len = 0;
bool allow_direct;
@ -864,7 +864,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
allow_direct = page_pool_napi_local(pool);
for (i = 0; i < count; i++) {
netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i]));
netmem_ref netmem = netmem_compound_head(data[i]);
/* It is not the last user for the page frag case */
if (!page_pool_is_last_ref(netmem))
@ -873,7 +873,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
netmem = __page_pool_put_page(pool, netmem, -1, allow_direct);
/* Approved for bulk recycling in ptr_ring cache */
if (netmem)
data[bulk_len++] = (__force void *)netmem;
data[bulk_len++] = netmem;
}
if (!bulk_len)
@ -882,7 +882,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
/* Bulk producer into ptr_ring page_pool cache */
in_softirq = page_pool_producer_lock(pool);
for (i = 0; i < bulk_len; i++) {
if (__ptr_ring_produce(&pool->ring, data[i])) {
if (__ptr_ring_produce(&pool->ring, (__force void *)data[i])) {
/* ring full */
recycle_stat_inc(pool, ring_full);
break;
@ -899,9 +899,9 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
* since put_page() with refcnt == 1 can be an expensive operation
*/
for (; i < bulk_len; i++)
page_pool_return_page(pool, (__force netmem_ref)data[i]);
page_pool_return_page(pool, data[i]);
}
EXPORT_SYMBOL(page_pool_put_page_bulk);
EXPORT_SYMBOL(page_pool_put_netmem_bulk);
static netmem_ref page_pool_drain_frag(struct page_pool *pool,
netmem_ref netmem)

6
net/core/xdp.c
View File

@ -518,7 +518,7 @@ void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq)
if (unlikely(!xa || !bq->count))
return;
page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count);
page_pool_put_netmem_bulk(xa->page_pool, bq->q, bq->count);
/* bq->xa is not cleared to save lookup, if mem.id same in next bulk */
bq->count = 0;
}
@ -559,12 +559,12 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf,
for (i = 0; i < sinfo->nr_frags; i++) {
skb_frag_t *frag = &sinfo->frags[i];
bq->q[bq->count++] = skb_frag_address(frag);
bq->q[bq->count++] = skb_frag_netmem(frag);
if (bq->count == XDP_BULK_QUEUE_SIZE)
xdp_flush_frame_bulk(bq);
}
}
bq->q[bq->count++] = xdpf->data;
bq->q[bq->count++] = virt_to_netmem(xdpf->data);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);