github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-31 10:33:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

xdp: Use nested-BH locking for system_page_pool

Browse Source 
system_page_pool is a per-CPU variable and relies on disabled BH for its
locking. Without per-CPU locking in local_bh_disable() on PREEMPT_RT
this data structure requires explicit locking.

Make a struct with a page_pool member (original system_page_pool) and a
local_lock_t and use local_lock_nested_bh() for locking. This change
adds only lockdep coverage and does not alter the functional behaviour
for !PREEMPT_RT.

Cc: Andrew Lunn <andrew+netdev@lunn.ch>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jesper Dangaard Brouer <hawk@kernel.org>
Cc: John Fastabend <john.fastabend@gmail.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://patch.msgid.link/20250512092736.229935-6-bigeasy@linutronix.de
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Sebastian Andrzej Siewior 2025-05-12 11:27:26 +02:00 committed by Paolo Abeni
parent bc57eda646
commit b9eef3391d
3 changed files with 26 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/linux/netdevice.h
View File

@ -3503,7 +3503,12 @@ struct softnet_data {
};
DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
DECLARE_PER_CPU(struct page_pool *, system_page_pool);
struct page_pool_bh {
struct page_pool *pool;
local_lock_t bh_lock;
};
DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);
#ifndef CONFIG_PREEMPT_RT
static inline int dev_recursion_level(void)

15
net/core/dev.c
View File

@ -462,7 +462,9 @@ EXPORT_PER_CPU_SYMBOL(softnet_data);
* PP consumers must pay attention to run APIs in the appropriate context
* (e.g. NAPI context).
*/
DEFINE_PER_CPU(struct page_pool *, system_page_pool);
DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {
.bh_lock = INIT_LOCAL_LOCK(bh_lock),
};
#ifdef CONFIG_LOCKDEP
/*
@ -5322,7 +5324,10 @@ netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog)
struct sk_buff *skb = *pskb;
int err, hroom, troom;
if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))
local_lock_nested_bh(&system_page_pool.bh_lock);
err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog);
local_unlock_nested_bh(&system_page_pool.bh_lock);
if (!err)
return 0;
/* In case we have to go down the path and also linearize,
@ -12712,7 +12717,7 @@ static int net_page_pool_create(int cpuid)
return err;
}
per_cpu(system_page_pool, cpuid) = pp_ptr;
per_cpu(system_page_pool.pool, cpuid) = pp_ptr;
#endif
return 0;
}
@ -12842,13 +12847,13 @@ static int __init net_dev_init(void)
for_each_possible_cpu(i) {
struct page_pool *pp_ptr;
pp_ptr = per_cpu(system_page_pool, i);
pp_ptr = per_cpu(system_page_pool.pool, i);
if (!pp_ptr)
continue;
xdp_unreg_page_pool(pp_ptr);
page_pool_destroy(pp_ptr);
per_cpu(system_page_pool, i) = NULL;
per_cpu(system_page_pool.pool, i) = NULL;
}
}

15
net/core/xdp.c
View File

@ -739,25 +739,27 @@ static noinline bool xdp_copy_frags_from_zc(struct sk_buff *skb,
*/
struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp)
{
struct page_pool *pp = this_cpu_read(system_page_pool);
const struct xdp_rxq_info *rxq = xdp->rxq;
u32 len = xdp->data_end - xdp->data_meta;
u32 truesize = xdp->frame_sz;
struct sk_buff *skb;
struct sk_buff *skb = NULL;
struct page_pool *pp;
int metalen;
void *data;
if (!IS_ENABLED(CONFIG_PAGE_POOL))
return NULL;
local_lock_nested_bh(&system_page_pool.bh_lock);
pp = this_cpu_read(system_page_pool.pool);
data = page_pool_dev_alloc_va(pp, &truesize);
if (unlikely(!data))
return NULL;
goto out;
skb = napi_build_skb(data, truesize);
if (unlikely(!skb)) {
page_pool_free_va(pp, data, true);
return NULL;
goto out;
}
skb_mark_for_recycle(skb);
@ -776,13 +778,16 @@ struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp)
if (unlikely(xdp_buff_has_frags(xdp)) &&
unlikely(!xdp_copy_frags_from_zc(skb, xdp, pp))) {
napi_consume_skb(skb, true);
return NULL;
skb = NULL;
goto out;
}
xsk_buff_free(xdp);
skb->protocol = eth_type_trans(skb, rxq->dev);
out:
local_unlock_nested_bh(&system_page_pool.bh_lock);
return skb;
}
EXPORT_SYMBOL_GPL(xdp_build_skb_from_zc);