github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-03 03:53:37 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'eth-fbnic-support-queue-api-and-zero-copy-rx'

Browse Source 
Jakub Kicinski says:

====================
eth: fbnic: support queue API and zero-copy Rx

Add support for queue API to fbnic, enable zero-copy Rx.

Patch 10 is likely of most interest as it adds a new core helper
(and touches mlx5). The rest of the patches are fbnic-specific
(and relatively boring).

Patches 1-3 reshuffle the Rx init/allocation path to better
align structures and functions which operate on them. Notably
patch 1 moves the page pool pointer to the queue struct (from NAPI).

Patch 4 converts the driver to use netmem_ref. The driver has
separate and explicit buffer queue for scatter / payloads, so only
references to those are converted.

Next 5 patches are more boring code shifts.

Patch 11 adds unreadable memory support to page pool allocation.

Patch 14 finally adds the support for queue API.

v2: https://lore.kernel.org/20250829012304.4146195-1-kuba@kernel.org
v1: https://lore.kernel.org/20250820025704.166248-1-kuba@kernel.org
====================

Link: https://patch.msgid.link/20250901211214.1027927-1-kuba@kernel.org
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni 2025-09-04 10:20:05 +02:00
commit cc621faa9c
8 changed files with 454 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@ -780,13 +780,6 @@ static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
bitmap_free(rq->mpwqe.shampo->bitmap);
}
static bool mlx5_rq_needs_separate_hd_pool(struct mlx5e_rq *rq)
{
struct netdev_rx_queue *rxq = __netif_get_rx_queue(rq->netdev, rq->ix);
return !!rxq->mp_params.mp_ops;
}
static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_param *rqp,
@ -825,7 +818,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
if (mlx5_rq_needs_separate_hd_pool(rq)) {
if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) {
/* Separate page pool for shampo headers */
struct page_pool_params pp_params = { };

1
drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
View File

@ -711,6 +711,7 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
netdev->netdev_ops = &fbnic_netdev_ops;
netdev->stat_ops = &fbnic_stat_ops;
netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops;
fbnic_set_ethtool_ops(netdev);

9
drivers/net/ethernet/meta/fbnic/fbnic_pci.c
View File

@ -206,8 +206,11 @@ static void fbnic_service_task(struct work_struct *work)
fbnic_bmc_rpc_check(fbd);
if (netif_carrier_ok(fbd->netdev))
if (netif_carrier_ok(fbd->netdev)) {
netdev_lock(fbd->netdev);
fbnic_napi_depletion_check(fbd->netdev);
netdev_unlock(fbd->netdev);
}
if (netif_running(fbd->netdev))
schedule_delayed_work(&fbd->service_task, HZ);
@ -391,12 +394,14 @@ static int fbnic_pm_suspend(struct device *dev)
goto null_uc_addr;
rtnl_lock();
netdev_lock(netdev);
netif_device_detach(netdev);
if (netif_running(netdev))
netdev->netdev_ops->ndo_stop(netdev);
netdev_unlock(netdev);
rtnl_unlock();
null_uc_addr:
@ -461,10 +466,12 @@ static int __fbnic_pm_resume(struct device *dev)
fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
rtnl_lock();
netdev_lock(netdev);
if (netif_running(netdev))
err = __fbnic_open(fbn);
netdev_unlock(netdev);
rtnl_unlock();
if (err)
goto err_free_mbx;

621
drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
View File

@ -640,7 +640,7 @@ static void fbnic_clean_twq1(struct fbnic_napi_vector *nv, bool pp_allow_direct,
FBNIC_TWD_TYPE_AL;
total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd);
page_pool_put_page(nv->page_pool, page, -1, pp_allow_direct);
page_pool_put_page(page->pp, page, -1, pp_allow_direct);
next_desc:
head++;
head &= ring->size_mask;
@ -715,35 +715,47 @@ static void fbnic_clean_tsq(struct fbnic_napi_vector *nv,
}
static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx,
struct page *page)
netmem_ref netmem)
{
struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx];
page_pool_fragment_page(page, FBNIC_PAGECNT_BIAS_MAX);
page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX);
rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX;
rx_buf->page = page;
rx_buf->netmem = netmem;
}
static struct page *fbnic_page_pool_get(struct fbnic_ring *ring,
unsigned int idx)
static struct page *
fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx)
{
struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx];
struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx];
rx_buf->pagecnt_bias--;
return rx_buf->page;
/* sub0 is always fed system pages, from the NAPI-level page_pool */
return netmem_to_page(rx_buf->netmem);
}
static netmem_ref
fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx)
{
struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx];
rx_buf->pagecnt_bias--;
return rx_buf->netmem;
}
static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx,
struct fbnic_napi_vector *nv, int budget)
int budget)
{
struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx];
struct page *page = rx_buf->page;
netmem_ref netmem = rx_buf->netmem;
if (!page_pool_unref_page(page, rx_buf->pagecnt_bias))
page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget);
if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias))
page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1,
!!budget);
rx_buf->page = NULL;
rx_buf->netmem = 0;
}
static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget,
@ -826,8 +838,8 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt,
fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1);
}
static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,
struct fbnic_ring *ring, unsigned int hw_head)
static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head,
int napi_budget)
{
unsigned int head = ring->head;
@ -835,7 +847,7 @@ static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,
return;
do {
fbnic_page_pool_drain(ring, head, nv, napi_budget);
fbnic_page_pool_drain(ring, head, napi_budget);
head++;
head &= ring->size_mask;
@ -844,10 +856,10 @@ static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,
ring->head = head;
}
static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page)
static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem)
{
__le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT];
dma_addr_t dma = page_pool_get_dma_addr(page);
dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem);
u64 bd, i = FBNIC_BD_FRAG_COUNT;
bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) |
@ -865,7 +877,7 @@ static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page)
} while (--i);
}
static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq)
static void fbnic_fill_bdq(struct fbnic_ring *bdq)
{
unsigned int count = fbnic_desc_unused(bdq);
unsigned int i = bdq->tail;
@ -874,10 +886,10 @@ static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq)
return;
do {
struct page *page;
netmem_ref netmem;
page = page_pool_dev_alloc_pages(nv->page_pool);
if (!page) {
netmem = page_pool_dev_alloc_netmems(bdq->page_pool);
if (!netmem) {
u64_stats_update_begin(&bdq->stats.syncp);
bdq->stats.rx.alloc_failed++;
u64_stats_update_end(&bdq->stats.syncp);
@ -885,8 +897,8 @@ static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq)
break;
}
fbnic_page_pool_init(bdq, i, page);
fbnic_bd_prep(bdq, i, page);
fbnic_page_pool_init(bdq, i, netmem);
fbnic_bd_prep(bdq, i, netmem);
i++;
i &= bdq->size_mask;
@ -933,7 +945,7 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd,
{
unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd);
unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd);
struct page *page = fbnic_page_pool_get(&qt->sub0, hdr_pg_idx);
struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx);
unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd);
unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom;
unsigned char *hdr_start;
@ -974,7 +986,7 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,
unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd);
unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd);
unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd);
struct page *page = fbnic_page_pool_get(&qt->sub1, pg_idx);
netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx);
unsigned int truesize;
bool added;
@ -985,11 +997,10 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,
FBNIC_BD_FRAG_SIZE;
/* Sync DMA buffer */
dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page),
pg_off, truesize, DMA_BIDIRECTIONAL);
page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem,
pg_off, truesize);
added = xdp_buff_add_frag(&pkt->buff, page_to_netmem(page), pg_off, len,
truesize);
added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize);
if (unlikely(!added)) {
pkt->add_frag_failed = true;
netdev_err_once(nv->napi.dev,
@ -997,7 +1008,7 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,
}
}
static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv,
static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt,
struct fbnic_pkt_buff *pkt, int budget)
{
struct page *page;
@ -1007,19 +1018,21 @@ static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv,
if (xdp_buff_has_frags(&pkt->buff)) {
struct skb_shared_info *shinfo;
netmem_ref netmem;
int nr_frags;
shinfo = xdp_get_shared_info_from_buff(&pkt->buff);
nr_frags = shinfo->nr_frags;
while (nr_frags--) {
page = skb_frag_page(&shinfo->frags[nr_frags]);
page_pool_put_full_page(nv->page_pool, page, !!budget);
netmem = skb_frag_netmem(&shinfo->frags[nr_frags]);
page_pool_put_full_netmem(qt->sub1.page_pool, netmem,
!!budget);
}
}
page = virt_to_page(pkt->buff.data_hard_start);
page_pool_put_full_page(nv->page_pool, page, !!budget);
page_pool_put_full_page(qt->sub0.page_pool, page, !!budget);
}
static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv,
@ -1274,7 +1287,7 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
dropped++;
}
fbnic_put_pkt_buff(nv, pkt, 1);
fbnic_put_pkt_buff(qt, pkt, 1);
}
pkt->buff.data_hard_start = NULL;
@ -1307,12 +1320,12 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
/* Unmap and free processed buffers */
if (head0 >= 0)
fbnic_clean_bdq(nv, budget, &qt->sub0, head0);
fbnic_fill_bdq(nv, &qt->sub0);
fbnic_clean_bdq(&qt->sub0, head0, budget);
fbnic_fill_bdq(&qt->sub0);
if (head1 >= 0)
fbnic_clean_bdq(nv, budget, &qt->sub1, head1);
fbnic_fill_bdq(nv, &qt->sub1);
fbnic_clean_bdq(&qt->sub1, head1, budget);
fbnic_fill_bdq(&qt->sub1);
/* Record the current head/tail of the queue */
if (rcq->head != head) {
@ -1462,6 +1475,12 @@ static void fbnic_remove_rx_ring(struct fbnic_net *fbn,
fbn->rx[rxr->q_idx] = NULL;
}
static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt)
{
page_pool_destroy(qt->sub0.page_pool);
page_pool_destroy(qt->sub1.page_pool);
}
static void fbnic_free_napi_vector(struct fbnic_net *fbn,
struct fbnic_napi_vector *nv)
{
@ -1475,15 +1494,13 @@ static void fbnic_free_napi_vector(struct fbnic_net *fbn,
}
for (j = 0; j < nv->rxt_count; j++, i++) {
xdp_rxq_info_unreg(&nv->qt[i].xdp_rxq);
fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0);
fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1);
fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl);
}
fbnic_napi_free_irq(fbd, nv);
page_pool_destroy(nv->page_pool);
netif_napi_del(&nv->napi);
netif_napi_del_locked(&nv->napi);
fbn->napi[fbnic_napi_idx(nv)] = NULL;
kfree(nv);
}
@ -1497,23 +1514,22 @@ void fbnic_free_napi_vectors(struct fbnic_net *fbn)
fbnic_free_napi_vector(fbn, fbn->napi[i]);
}
#define FBNIC_PAGE_POOL_FLAGS \
(PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV)
static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn,
struct fbnic_napi_vector *nv)
static int
fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt,
unsigned int rxq_idx)
{
struct page_pool_params pp_params = {
.order = 0,
.flags = FBNIC_PAGE_POOL_FLAGS,
.pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count,
.flags = PP_FLAG_DMA_MAP |
PP_FLAG_DMA_SYNC_DEV,
.pool_size = fbn->hpq_size + fbn->ppq_size,
.nid = NUMA_NO_NODE,
.dev = nv->dev,
.dev = fbn->netdev->dev.parent,
.dma_dir = DMA_BIDIRECTIONAL,
.offset = 0,
.max_len = PAGE_SIZE,
.napi = &nv->napi,
.netdev = fbn->netdev,
.queue_idx = rxq_idx,
};
struct page_pool *pp;
@ -1533,9 +1549,24 @@ static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn,
if (IS_ERR(pp))
return PTR_ERR(pp);
nv->page_pool = pp;
qt->sub0.page_pool = pp;
if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) {
pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM;
pp_params.dma_dir = DMA_FROM_DEVICE;
pp = page_pool_create(&pp_params);
if (IS_ERR(pp))
goto err_destroy_sub0;
} else {
page_pool_get(pp);
}
qt->sub1.page_pool = pp;
return 0;
err_destroy_sub0:
page_pool_destroy(pp);
return PTR_ERR(pp);
}
static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell,
@ -1590,26 +1621,20 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
/* Tie napi to netdev */
fbn->napi[fbnic_napi_idx(nv)] = nv;
netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll);
netif_napi_add_locked(fbn->netdev, &nv->napi, fbnic_poll);
/* Record IRQ to NAPI struct */
netif_napi_set_irq(&nv->napi,
pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx));
netif_napi_set_irq_locked(&nv->napi,
pci_irq_vector(to_pci_dev(fbd->dev),
nv->v_idx));
/* Tie nv back to PCIe dev */
nv->dev = fbd->dev;
/* Allocate page pool */
if (rxq_count) {
err = fbnic_alloc_nv_page_pool(fbn, nv);
if (err)
goto napi_del;
}
/* Request the IRQ for napi vector */
err = fbnic_napi_request_irq(fbd, nv);
if (err)
goto pp_destroy;
goto napi_del;
/* Initialize queue triads */
qt = nv->qt;
@ -1679,11 +1704,6 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS);
fbn->rx[rxq_idx] = &qt->cmpl;
err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, rxq_idx,
nv->napi.napi_id);
if (err)
goto free_ring_cur_qt;
/* Update Rx queue index */
rxt_count--;
rxq_idx += v_count;
@ -1694,30 +1714,8 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
return 0;
while (rxt_count < nv->rxt_count) {
qt--;
xdp_rxq_info_unreg(&qt->xdp_rxq);
free_ring_cur_qt:
fbnic_remove_rx_ring(fbn, &qt->sub0);
fbnic_remove_rx_ring(fbn, &qt->sub1);
fbnic_remove_rx_ring(fbn, &qt->cmpl);
rxt_count++;
}
while (txt_count < nv->txt_count) {
qt--;
fbnic_remove_tx_ring(fbn, &qt->sub0);
fbnic_remove_xdp_ring(fbn, &qt->sub1);
fbnic_remove_tx_ring(fbn, &qt->cmpl);
txt_count++;
}
fbnic_napi_free_irq(fbd, nv);
pp_destroy:
page_pool_destroy(nv->page_pool);
napi_del:
netif_napi_del(&nv->napi);
netif_napi_del_locked(&nv->napi);
fbn->napi[fbnic_napi_idx(nv)] = NULL;
kfree(nv);
return err;
@ -1931,6 +1929,12 @@ static void fbnic_free_qt_resources(struct fbnic_net *fbn,
fbnic_free_ring_resources(dev, &qt->cmpl);
fbnic_free_ring_resources(dev, &qt->sub1);
fbnic_free_ring_resources(dev, &qt->sub0);
if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) {
xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq);
xdp_rxq_info_unreg(&qt->xdp_rxq);
fbnic_free_qt_page_pools(qt);
}
}
static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn,
@ -1961,15 +1965,30 @@ static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn,
}
static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn,
struct fbnic_napi_vector *nv,
struct fbnic_q_triad *qt)
{
struct device *dev = fbn->netdev->dev.parent;
int err;
err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0);
err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx);
if (err)
return err;
err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx,
nv->napi.napi_id);
if (err)
goto free_page_pools;
err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL,
qt->sub0.page_pool);
if (err)
goto unreg_rxq;
err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0);
if (err)
goto unreg_mm;
err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1);
if (err)
goto free_sub0;
@ -1984,22 +2003,22 @@ static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn,
fbnic_free_ring_resources(dev, &qt->sub1);
free_sub0:
fbnic_free_ring_resources(dev, &qt->sub0);
unreg_mm:
xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq);
unreg_rxq:
xdp_rxq_info_unreg(&qt->xdp_rxq);
free_page_pools:
fbnic_free_qt_page_pools(qt);
return err;
}
static void fbnic_free_nv_resources(struct fbnic_net *fbn,
struct fbnic_napi_vector *nv)
{
int i, j;
int i;
/* Free Tx Resources */
for (i = 0; i < nv->txt_count; i++)
for (i = 0; i < nv->txt_count + nv->rxt_count; i++)
fbnic_free_qt_resources(fbn, &nv->qt[i]);
for (j = 0; j < nv->rxt_count; j++, i++) {
fbnic_free_qt_resources(fbn, &nv->qt[i]);
xdp_rxq_info_unreg_mem_model(&nv->qt[i].xdp_rxq);
}
}
static int fbnic_alloc_nv_resources(struct fbnic_net *fbn,
@ -2016,26 +2035,13 @@ static int fbnic_alloc_nv_resources(struct fbnic_net *fbn,
/* Allocate Rx Resources */
for (j = 0; j < nv->rxt_count; j++, i++) {
/* Register XDP memory model for completion queue */
err = xdp_reg_mem_model(&nv->qt[i].xdp_rxq.mem,
MEM_TYPE_PAGE_POOL,
nv->page_pool);
err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]);
if (err)
goto xdp_unreg_mem_model;
err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]);
if (err)
goto xdp_unreg_cur_model;
goto free_qt_resources;
}
return 0;
xdp_unreg_mem_model:
while (j-- && i--) {
fbnic_free_qt_resources(fbn, &nv->qt[i]);
xdp_unreg_cur_model:
xdp_rxq_info_unreg_mem_model(&nv->qt[i].xdp_rxq);
}
free_qt_resources:
while (i--)
fbnic_free_qt_resources(fbn, &nv->qt[i]);
@ -2178,37 +2184,48 @@ void fbnic_napi_disable(struct fbnic_net *fbn)
int i;
for (i = 0; i < fbn->num_napi; i++) {
napi_disable(&fbn->napi[i]->napi);
napi_disable_locked(&fbn->napi[i]->napi);
fbnic_nv_irq_disable(fbn->napi[i]);
}
}
static void __fbnic_nv_disable(struct fbnic_napi_vector *nv)
{
int i, t;
/* Disable Tx queue triads */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_disable_twq0(&qt->sub0);
fbnic_disable_twq1(&qt->sub1);
fbnic_disable_tcq(&qt->cmpl);
}
/* Disable Rx queue triads */
for (i = 0; i < nv->rxt_count; i++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_disable_bdq(&qt->sub0, &qt->sub1);
fbnic_disable_rcq(&qt->cmpl);
}
}
static void
fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv)
{
__fbnic_nv_disable(nv);
fbnic_wrfl(fbn->fbd);
}
void fbnic_disable(struct fbnic_net *fbn)
{
struct fbnic_dev *fbd = fbn->fbd;
int i, j, t;
int i;
for (i = 0; i < fbn->num_napi; i++) {
struct fbnic_napi_vector *nv = fbn->napi[i];
/* Disable Tx queue triads */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_disable_twq0(&qt->sub0);
fbnic_disable_twq1(&qt->sub1);
fbnic_disable_tcq(&qt->cmpl);
}
/* Disable Rx queue triads */
for (j = 0; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_disable_bdq(&qt->sub0, &qt->sub1);
fbnic_disable_rcq(&qt->cmpl);
}
}
for (i = 0; i < fbn->num_napi; i++)
__fbnic_nv_disable(fbn->napi[i]);
fbnic_wrfl(fbd);
}
@ -2297,51 +2314,108 @@ int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail)
return err;
}
static int
fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx)
{
static const unsigned int tx_regs[] = {
FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0),
FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0),
}, rx_regs[] = {
FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0),
FBNIC_QM_RCQ_IDLE(0),
};
struct fbnic_dev *fbd = fbn->fbd;
unsigned int val, mask, off;
const unsigned int *regs;
unsigned int reg_cnt;
int i, err;
regs = rx ? rx_regs : tx_regs;
reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs);
off = idx / 32;
mask = BIT(idx % 32);
for (i = 0; i < reg_cnt; i++) {
err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask,
2, 500000, false,
fbd, regs[i] + off);
if (err) {
netdev_err(fbd->netdev,
"wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n",
rx ? "Rx" : "Tx", idx, regs[i] + off, i,
val, mask);
return err;
}
}
return 0;
}
static void fbnic_nv_flush(struct fbnic_napi_vector *nv)
{
int j, t;
/* Flush any processed Tx Queue Triads and drop the rest */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
struct netdev_queue *tx_queue;
/* Clean the work queues of unprocessed work */
fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail);
fbnic_clean_twq1(nv, false, &qt->sub1, true,
qt->sub1.tail);
/* Reset completion queue descriptor ring */
memset(qt->cmpl.desc, 0, qt->cmpl.size);
/* Nothing else to do if Tx queue is disabled */
if (qt->sub0.flags & FBNIC_RING_F_DISABLED)
continue;
/* Reset BQL associated with Tx queue */
tx_queue = netdev_get_tx_queue(nv->napi.dev,
qt->sub0.q_idx);
netdev_tx_reset_queue(tx_queue);
}
/* Flush any processed Rx Queue Triads and drop the rest */
for (j = 0; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
/* Clean the work queues of unprocessed work */
fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0);
fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0);
/* Reset completion queue descriptor ring */
memset(qt->cmpl.desc, 0, qt->cmpl.size);
fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0);
memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff));
}
}
void fbnic_flush(struct fbnic_net *fbn)
{
int i;
for (i = 0; i < fbn->num_napi; i++) {
struct fbnic_napi_vector *nv = fbn->napi[i];
int j, t;
for (i = 0; i < fbn->num_napi; i++)
fbnic_nv_flush(fbn->napi[i]);
}
/* Flush any processed Tx Queue Triads and drop the rest */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
struct netdev_queue *tx_queue;
static void fbnic_nv_fill(struct fbnic_napi_vector *nv)
{
int j, t;
/* Clean the work queues of unprocessed work */
fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail);
fbnic_clean_twq1(nv, false, &qt->sub1, true,
qt->sub1.tail);
/* Configure NAPI mapping and populate pages
* in the BDQ rings to use for Rx
*/
for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
/* Reset completion queue descriptor ring */
memset(qt->cmpl.desc, 0, qt->cmpl.size);
/* Nothing else to do if Tx queue is disabled */
if (qt->sub0.flags & FBNIC_RING_F_DISABLED)
continue;
/* Reset BQL associated with Tx queue */
tx_queue = netdev_get_tx_queue(nv->napi.dev,
qt->sub0.q_idx);
netdev_tx_reset_queue(tx_queue);
}
/* Flush any processed Rx Queue Triads and drop the rest */
for (j = 0; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
/* Clean the work queues of unprocessed work */
fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail);
fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail);
/* Reset completion queue descriptor ring */
memset(qt->cmpl.desc, 0, qt->cmpl.size);
fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0);
memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff));
}
/* Populate the header and payload BDQs */
fbnic_fill_bdq(&qt->sub0);
fbnic_fill_bdq(&qt->sub1);
}
}
@ -2349,21 +2423,8 @@ void fbnic_fill(struct fbnic_net *fbn)
{
int i;
for (i = 0; i < fbn->num_napi; i++) {
struct fbnic_napi_vector *nv = fbn->napi[i];
int j, t;
/* Configure NAPI mapping and populate pages
* in the BDQ rings to use for Rx
*/
for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
/* Populate the header and payload BDQs */
fbnic_fill_bdq(nv, &qt->sub0);
fbnic_fill_bdq(nv, &qt->sub1);
}
}
for (i = 0; i < fbn->num_napi; i++)
fbnic_nv_fill(fbn->napi[i]);
}
static void fbnic_enable_twq0(struct fbnic_ring *twq)
@ -2581,33 +2642,47 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv,
fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE);
}
static void __fbnic_nv_enable(struct fbnic_napi_vector *nv)
{
int j, t;
/* Setup Tx Queue Triads */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_enable_twq0(&qt->sub0);
fbnic_enable_twq1(&qt->sub1);
fbnic_enable_tcq(nv, &qt->cmpl);
}
/* Setup Rx Queue Triads */
for (j = 0; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
page_pool_enable_direct_recycling(qt->sub0.page_pool,
&nv->napi);
page_pool_enable_direct_recycling(qt->sub1.page_pool,
&nv->napi);
fbnic_enable_bdq(&qt->sub0, &qt->sub1);
fbnic_config_drop_mode_rcq(nv, &qt->cmpl);
fbnic_enable_rcq(nv, &qt->cmpl);
}
}
static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv)
{
__fbnic_nv_enable(nv);
fbnic_wrfl(fbn->fbd);
}
void fbnic_enable(struct fbnic_net *fbn)
{
struct fbnic_dev *fbd = fbn->fbd;
int i;
for (i = 0; i < fbn->num_napi; i++) {
struct fbnic_napi_vector *nv = fbn->napi[i];
int j, t;
/* Setup Tx Queue Triads */
for (t = 0; t < nv->txt_count; t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_enable_twq0(&qt->sub0);
fbnic_enable_twq1(&qt->sub1);
fbnic_enable_tcq(nv, &qt->cmpl);
}
/* Setup Rx Queue Triads */
for (j = 0; j < nv->rxt_count; j++, t++) {
struct fbnic_q_triad *qt = &nv->qt[t];
fbnic_enable_bdq(&qt->sub0, &qt->sub1);
fbnic_config_drop_mode_rcq(nv, &qt->cmpl);
fbnic_enable_rcq(nv, &qt->cmpl);
}
}
for (i = 0; i < fbn->num_napi; i++)
__fbnic_nv_enable(fbn->napi[i]);
fbnic_wrfl(fbd);
}
@ -2626,7 +2701,7 @@ void fbnic_napi_enable(struct fbnic_net *fbn)
for (i = 0; i < fbn->num_napi; i++) {
struct fbnic_napi_vector *nv = fbn->napi[i];
napi_enable(&nv->napi);
napi_enable_locked(&nv->napi);
fbnic_nv_irq_enable(nv);
@ -2679,3 +2754,123 @@ void fbnic_napi_depletion_check(struct net_device *netdev)
fbnic_wrfl(fbd);
}
static int fbnic_queue_mem_alloc(struct net_device *dev, void *qmem, int idx)
{
struct fbnic_net *fbn = netdev_priv(dev);
const struct fbnic_q_triad *real;
struct fbnic_q_triad *qt = qmem;
struct fbnic_napi_vector *nv;
if (!netif_running(dev))
return fbnic_alloc_qt_page_pools(fbn, qt, idx);
real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
nv = fbn->napi[idx % fbn->num_napi];
fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx,
real->sub0.flags);
fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx,
real->sub1.flags);
fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx,
real->cmpl.flags);
return fbnic_alloc_rx_qt_resources(fbn, nv, qt);
}
static void fbnic_queue_mem_free(struct net_device *dev, void *qmem)
{
struct fbnic_net *fbn = netdev_priv(dev);
struct fbnic_q_triad *qt = qmem;
if (!netif_running(dev))
fbnic_free_qt_page_pools(qt);
else
fbnic_free_qt_resources(fbn, qt);
}
static void __fbnic_nv_restart(struct fbnic_net *fbn,
struct fbnic_napi_vector *nv)
{
struct fbnic_dev *fbd = fbn->fbd;
int i;
fbnic_nv_enable(fbn, nv);
fbnic_nv_fill(nv);
napi_enable_locked(&nv->napi);
fbnic_nv_irq_enable(nv);
fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32));
fbnic_wrfl(fbd);
for (i = 0; i < nv->txt_count; i++)
netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx);
}
static int fbnic_queue_start(struct net_device *dev, void *qmem, int idx)
{
struct fbnic_net *fbn = netdev_priv(dev);
struct fbnic_napi_vector *nv;
struct fbnic_q_triad *real;
real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
nv = fbn->napi[idx % fbn->num_napi];
fbnic_aggregate_ring_rx_counters(fbn, &real->sub0);
fbnic_aggregate_ring_rx_counters(fbn, &real->sub1);
fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl);
memcpy(real, qmem, sizeof(*real));
__fbnic_nv_restart(fbn, nv);
return 0;
}
static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx)
{
struct fbnic_net *fbn = netdev_priv(dev);
const struct fbnic_q_triad *real;
struct fbnic_napi_vector *nv;
int i, t;
int err;
real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
nv = fbn->napi[idx % fbn->num_napi];
napi_disable_locked(&nv->napi);
fbnic_nv_irq_disable(nv);
for (i = 0; i < nv->txt_count; i++)
netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx);
fbnic_nv_disable(fbn, nv);
for (t = 0; t < nv->txt_count + nv->rxt_count; t++) {
err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count,
nv->qt[t].sub0.q_idx);
if (err)
goto err_restart;
}
fbnic_synchronize_irq(fbn->fbd, nv->v_idx);
fbnic_nv_flush(nv);
page_pool_disable_direct_recycling(real->sub0.page_pool);
page_pool_disable_direct_recycling(real->sub1.page_pool);
memcpy(qmem, real, sizeof(*real));
return 0;
err_restart:
__fbnic_nv_restart(fbn, nv);
return err;
}
const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = {
.ndo_queue_mem_size = sizeof(struct fbnic_q_triad),
.ndo_queue_mem_alloc = fbnic_queue_mem_alloc,
.ndo_queue_mem_free = fbnic_queue_mem_free,
.ndo_queue_start = fbnic_queue_start,
.ndo_queue_stop = fbnic_queue_stop,
};

20
drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
View File

@ -100,7 +100,7 @@ struct fbnic_queue_stats {
#define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE
struct fbnic_rx_buf {
struct page *page;
netmem_ref netmem;
long pagecnt_bias;
};
@ -121,11 +121,16 @@ struct fbnic_ring {
u32 head, tail; /* Head/Tail of ring */
/* Deferred_head is used to cache the head for TWQ1 if an attempt
* is made to clean TWQ1 with zero napi_budget. We do not use it for
* any other ring.
*/
s32 deferred_head;
union {
/* Rx BDQs only */
struct page_pool *page_pool;
/* Deferred_head is used to cache the head for TWQ1 if
* an attempt is made to clean TWQ1 with zero napi_budget.
* We do not use it for any other ring.
*/
s32 deferred_head;
};
struct fbnic_queue_stats stats;
@ -142,7 +147,6 @@ struct fbnic_q_triad {
struct fbnic_napi_vector {
struct napi_struct napi;
struct device *dev; /* Device for DMA unmapping */
struct page_pool *page_pool;
struct fbnic_dev *fbd;
u16 v_idx;
@ -152,6 +156,8 @@ struct fbnic_napi_vector {
struct fbnic_q_triad qt[];
};
extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops;
netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev);
netdev_features_t
fbnic_features_check(struct sk_buff *skb, struct net_device *dev,

2
include/net/netdev_queues.h
View File

@ -151,6 +151,8 @@ struct netdev_queue_mgmt_ops {
int idx);
};
bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx);
/**
* DOC: Lockless queue stopping / waking helpers.
*

12
include/net/page_pool/helpers.h
View File

@ -505,6 +505,18 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
page_pool_update_nid(pool, new_nid);
}
/**
* page_pool_is_unreadable() - will allocated buffers be unreadable for the CPU
* @pool: queried page pool
*
* Check if page pool will return buffers which are unreadable to the CPU /
* kernel. This will only be the case if user space bound a memory provider (mp)
* which returns unreadable memory to the queue served by the page pool.
* If %PP_FLAG_ALLOW_UNREADABLE_NETMEM was set but there is no mp bound
* this helper will return false. See also netif_rxq_has_unreadable_mp().
*
* Return: true if memory allocated by the page pool may be unreadable
*/
static inline bool page_pool_is_unreadable(struct page_pool *pool)
{
return !!pool->mp_ops;

9
net/core/netdev_rx_queue.c
View File

@ -9,6 +9,15 @@
#include "page_pool_priv.h"
/* See also page_pool_is_unreadable() */
bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx)
{
struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, idx);
return !!rxq->mp_params.mp_ops;
}
EXPORT_SYMBOL(netif_rxq_has_unreadable_mp);
int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx)
{
struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx);