github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-12 16:18:45 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

xdp: make __xdp_return() MP-agnostic

Browse Source 
Currently, __xdp_return() takes pointer to the virtual memory to free
a buffer. Apart from that this sometimes provokes redundant
data <--> page conversions, taking data pointer effectively prevents
lots of XDP code to support non-page-backed buffers, as there's no
mapping for the non-host memory (data is always NULL).
Just convert it to always take netmem reference. For
xdp_return_{buff,frame*}(), this chops off one page_address() per each
frag and adds one virt_to_netmem() (same as virt_to_page()) per header
buffer. For __xdp_return() itself, it removes one virt_to_page() for
MEM_TYPE_PAGE_POOL and another one for MEM_TYPE_PAGE_ORDER0, adding
one page_address() for [not really common nowadays]
MEM_TYPE_PAGE_SHARED, but the main effect is that the abovementioned
functions won't die or memleak anymore if the frame has non-host memory
attached and will correctly free those.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241211172649.761483-4-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Alexander Lobakin 2024-12-11 18:26:40 +01:00 committed by Jakub Kicinski
parent 56d95b0adf
commit 207ff83cec
3 changed files with 24 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/net/xdp.h
View File

@ -312,8 +312,8 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
return xdp_frame;
}
void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
struct xdp_buff *xdp);
void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,
bool napi_direct, struct xdp_buff *xdp);
void xdp_return_frame(struct xdp_frame *xdpf);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
void xdp_return_buff(struct xdp_buff *xdp);

9
net/core/filter.c
View File

@ -4125,7 +4125,7 @@ static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
if (release) {
xsk_buff_del_tail(zc_frag);
__xdp_return(NULL, mem_type, false, zc_frag);
__xdp_return(0, mem_type, false, zc_frag);
} else {
zc_frag->data_end -= shrink;
}
@ -4142,11 +4142,8 @@ static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
goto out;
}
if (release) {
struct page *page = skb_frag_page(frag);
__xdp_return(page_address(page), mem_type, false, NULL);
}
if (release)
__xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);
out:
return release;

47
net/core/xdp.c
View File

@ -430,27 +430,25 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool);
* is used for those calls sites. Thus, allowing for faster recycling
* of xdp_frames/pages in those cases.
*/
void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
struct xdp_buff *xdp)
void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,
bool napi_direct, struct xdp_buff *xdp)
{
struct page *page;
switch (mem_type) {
case MEM_TYPE_PAGE_POOL:
page = virt_to_head_page(data);
netmem = netmem_compound_head(netmem);
if (napi_direct && xdp_return_frame_no_direct())
napi_direct = false;
/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
* as mem->type knows this a page_pool page
*/
page_pool_put_full_page(page->pp, page, napi_direct);
page_pool_put_full_netmem(netmem_get_pp(netmem), netmem,
napi_direct);
break;
case MEM_TYPE_PAGE_SHARED:
page_frag_free(data);
page_frag_free(__netmem_address(netmem));
break;
case MEM_TYPE_PAGE_ORDER0:
page = virt_to_page(data); /* Assumes order0 page*/
put_page(page);
put_page(__netmem_to_page(netmem));
break;
case MEM_TYPE_XSK_BUFF_POOL:
/* NB! Only valid from an xdp_buff! */
@ -466,38 +464,34 @@ void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
void xdp_return_frame(struct xdp_frame *xdpf)
{
struct skb_shared_info *sinfo;
int i;
if (likely(!xdp_frame_has_frags(xdpf)))
goto out;
sinfo = xdp_get_shared_info_from_frame(xdpf);
for (i = 0; i < sinfo->nr_frags; i++) {
struct page *page = skb_frag_page(&sinfo->frags[i]);
for (u32 i = 0; i < sinfo->nr_frags; i++)
__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,
false, NULL);
__xdp_return(page_address(page), xdpf->mem_type, false, NULL);
}
out:
__xdp_return(xdpf->data, xdpf->mem_type, false, NULL);
__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, false, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
{
struct skb_shared_info *sinfo;
int i;
if (likely(!xdp_frame_has_frags(xdpf)))
goto out;
sinfo = xdp_get_shared_info_from_frame(xdpf);
for (i = 0; i < sinfo->nr_frags; i++) {
struct page *page = skb_frag_page(&sinfo->frags[i]);
for (u32 i = 0; i < sinfo->nr_frags; i++)
__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,
true, NULL);
__xdp_return(page_address(page), xdpf->mem_type, true, NULL);
}
out:
__xdp_return(xdpf->data, xdpf->mem_type, true, NULL);
__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, true, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
@ -544,20 +538,17 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
void xdp_return_buff(struct xdp_buff *xdp)
{
struct skb_shared_info *sinfo;
int i;
if (likely(!xdp_buff_has_frags(xdp)))
goto out;
sinfo = xdp_get_shared_info_from_buff(xdp);
for (i = 0; i < sinfo->nr_frags; i++) {
struct page *page = skb_frag_page(&sinfo->frags[i]);
for (u32 i = 0; i < sinfo->nr_frags; i++)
__xdp_return(skb_frag_netmem(&sinfo->frags[i]),
xdp->rxq->mem.type, true, xdp);
__xdp_return(page_address(page), xdp->rxq->mem.type, true,
xdp);
}
out:
__xdp_return(xdp->data, xdp->rxq->mem.type, true, xdp);
__xdp_return(virt_to_netmem(xdp->data), xdp->rxq->mem.type, true, xdp);
}
EXPORT_SYMBOL_GPL(xdp_return_buff);