From 1b82958f3f035df5ccaab5430a2302f08a5d5351 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:09:57 -0700 Subject: [PATCH 1/6] net: ethtool: keep rtnl_lock for the ioctl self test An offline self test that brings the interface down and back up with netif_close() / netif_open() requires rtnl_lock for both. Since the ethtool IOCTL path became rtnl-optional for ops-locked drivers, the ETHTOOL_TEST ioctl runs holding only the netdev instance lock, so on an ops-locked driver the self test now tears the device down without rtnl_lock. With lockdep this reproduces deterministically on every offline self test on such a driver; note the sole lock held is the instance lock, not rtnl: WARNING: suspicious RCU usage net/core/netpoll.c:207 suspicious rcu_dereference_protected() usage! 1 lock held by ethtool/107: #0: (&dev->lock){+.+.}, at: dev_ethtool Call Trace: netpoll_poll_disable __dev_close_many netif_close_many netif_close fbnic_self_test dev_ethtool_locked dev_ethtool dev_ioctl sock_ioctl __x64_sys_ioctl Without lockdep the same condition trips ASSERT_RTNL() in __dev_close_many() / __dev_open(); that check only samples the global rtnl state, so it can be masked by a concurrent rtnl holder, but the device is still being reconfigured without the lock it requires. The ethtool self_test is a legacy ioctl-only command, so an ETHTOOL_TEST case is only needed on the ioctl path. Add an opt-in bit for drivers whose self test needs rtnl_lock and set it on the ops-locked drivers whose offline self test tears the interface down and up: - fbnic (ops-locked via queue_mgmt_ops): fbnic_self_test() offline path uses netif_close() / netif_open(). - bnxt (ops-locked via queue_mgmt_ops): bnxt_self_test() offline path goes through bnxt_close_nic() / bnxt_half_open_nic() / bnxt_half_close_nic() / bnxt_open_nic(), which close and reopen the device. Fixes: f994752b1127 ("net: ethtool: optionally skip rtnl_lock on IOCTL path") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942019771.7700.338431553546884773.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 3 ++- drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c | 3 ++- include/linux/ethtool.h | 2 ++ net/ethtool/common.h | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 62bc9cae613c..622e89587e5d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -5733,7 +5733,8 @@ const struct ethtool_ops bnxt_ethtool_ops = { .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | ETHTOOL_OP_NEEDS_RTNL_SCOALESCE | - ETHTOOL_OP_NEEDS_RTNL_RSS, + ETHTOOL_OP_NEEDS_RTNL_RSS | + ETHTOOL_OP_NEEDS_RTNL_TEST, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USECS_IRQ | diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 0e47088ec44b..423f179c9d47 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -2025,7 +2025,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = { ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM | ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | - ETHTOOL_OP_NEEDS_RTNL_GLINK, + ETHTOOL_OP_NEEDS_RTNL_GLINK | + ETHTOOL_OP_NEEDS_RTNL_TEST, .get_drvinfo = fbnic_get_drvinfo, .get_regs_len = fbnic_get_regs_len, .get_regs = fbnic_get_regs, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 253600c0eccd..c4c9ce038611 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -944,6 +944,7 @@ struct kernel_ethtool_ts_info { #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) #define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) #define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) +#define ETHTOOL_OP_NEEDS_RTNL_TEST BIT(9) /** * struct ethtool_ops - optional netdev operations @@ -981,6 +982,7 @@ struct kernel_ethtool_ts_info { * - netdev_update_features() * - netif_set_real_num_tx_queues() * - ethtool_op_get_link() (syncs link watch under rtnl_lock) + * - netif_open() / netif_close() (used by @self_test) * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 4e5356e26f40..ae32e7fdb563 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -163,6 +163,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd) return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; case ETHTOOL_GLINK: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; + case ETHTOOL_TEST: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_TEST; } return false; } From 1f4c73064a50f53d596c6f1d06d2d700f43c4b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Mon, 14 Sep 2026 14:10:04 -0700 Subject: [PATCH 2/6] eth: fbnic: Handle maximum standalone channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone channels use one NAPI vector for each Tx and Rx queue. fbnic's allocation path excludes FBNIC_MAX_TXQS from that layout. A 64-Tx/64-Rx configuration therefore records 128 vectors but allocates only 64, leaving NULL entries that resource setup dereferences. Include the maximum vector count in standalone allocation. Fixes: bc6107771bb4 ("eth: fbnic: Allocate a netdevice and napi vectors with queues") Signed-off-by: Björn Töpel Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942020457.7700.13129750616387075931.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/meta/fbnic/fbnic_txrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 661dee1661af..a30aa4450848 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -1791,7 +1791,7 @@ int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) int err; /* Allocate 1 Tx queue per napi vector */ - if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { + if (num_napi <= FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { while (num_tx) { err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, From b5d9e9d4d0c13bc8b60d8d97e7a07fb25fea639e Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:10:11 -0700 Subject: [PATCH 3/6] eth: fbnic: use the Rx queue napi pointer to find the napi vector The queue management ndos pick the napi vector for an Rx queue with: nv = fbn->napi[idx % fbn->num_napi]; The issue is this is only correct in the cases where there are no standalone Tx vectors. In those cases we were allocating the Tx vectors first and then the Rx so the queues would be pointing to Tx NAPI vectors instead of the Rx ones. The mapping the ndos want is already recorded. fbnic_set_netif_napi() publishes it with netif_queue_set_napi(), which stores the napi pointer in netdev_rx_queue.napi, and fbnic_reset_netif_napi() clears it again. Both run under the netdev instance lock that the queue management ndos also hold, so the pointer can be read directly. Use it and drop the divide. The pointer is NULL exactly while the datapath is down, so fbnic_queue_mem_alloc() can reject that case rather than reaching into freed state: netdev_rx_queue_restart() calls it before it tests netif_running(), and fbnic_pm_suspend() leaves netif_running() true across a PCIe recovery that never completes, so a queue restart can arrive after fbnic_stop() has freed the rings and the vectors. fbnic_stop() clears the association in fbnic_reset_netif_queues() before fbnic_free_napi_vectors(), so the NULL is always published first. fbnic_queue_start() and fbnic_queue_stop() need no check of their own, as netdev_rx_queue_reconfig() only reaches them once fbnic_queue_mem_alloc() has succeeded under the same instance lock. Fixes: da43127a8edc ("eth: fbnic: support queue ops / zero-copy Rx") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942021136.7700.4391219358260544104.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/meta/fbnic/fbnic_txrx.c | 26 +++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index a30aa4450848..10caacffee0f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -2853,6 +2854,17 @@ void fbnic_napi_depletion_check(struct net_device *netdev) fbnic_wrfl(fbd); } +/* Returns the napi vector servicing an Rx queue, or NULL if the datapath + * is torn down. The association is published by fbnic_set_netif_napi() + * and cleared by fbnic_reset_netif_napi(), both under the instance lock. + */ +static struct fbnic_napi_vector *fbnic_rxq_nv(struct net_device *dev, int idx) +{ + struct napi_struct *napi = __netif_get_rx_queue(dev, idx)->napi; + + return napi ? container_of(napi, struct fbnic_napi_vector, napi) : NULL; +} + static int fbnic_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *qmem, int idx) @@ -2865,8 +2877,16 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, if (!netif_running(dev)) return fbnic_alloc_qt_page_pools(fbn, qt, idx); + /* A failed PCIe recovery or resume can leave the datapath torn down + * while netif_running() is still true. This ndo runs before + * netdev_rx_queue_restart() checks netif_running(), so bail out + * rather than touching rings and vectors that are already freed. + */ + nv = fbnic_rxq_nv(dev, idx); + if (!nv) + return -ENETDOWN; + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, real->sub0.flags); @@ -2917,7 +2937,7 @@ static int fbnic_queue_start(struct net_device *dev, struct fbnic_q_triad *real; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); @@ -2939,7 +2959,7 @@ static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) int err; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_dbg_nv_exit(nv); napi_disable_locked(&nv->napi); From 4bcc4a92c603fe7f062cea22e20da2e0ad6b12c3 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:10:18 -0700 Subject: [PATCH 4/6] eth: fbnic: reset num_napi when the napi vectors are freed fbn->num_napi is the count of live napi vectors, each of which owns an IRQ. The PM path had freed them without clearing the count. fbnic_pm_suspend() tears the datapath down via ndo_stop() and frees the IRQs, but leaves netif_running() true so resume knows to re-open. Resume rebuilds the datapath in __fbnic_pm_resume() and fbnic_reset_queues() sets num_napi and __fbnic_open() re-allocates the vectors. When the datapath is torn down but never rebuilt, num_napi is left pointing at freed vectors under 2 different scenarios: - a PCIe error recovery that fails (fbnic_err_slot_reset() -> __fbnic_pm_resume() returns an error -> PCI_ERS_RESULT_DISCONNECT), so .resume never runs; or - an __fbnic_open() that fails partway on resume and unwinds, freeing the vectors after fbnic_reset_queues() has already set num_napi. The netdev is then running with num_napi > 0 but napi[] freed, and the eventual remove/unbind close re-enters fbnic_down() -> fbnic_dbg_down() and dereferences the freed vectors: BUG: kernel NULL pointer dereference, address: 0000000000000210 RIP: fbnic_dbg_down+0x28 Clear num_napi when the vectors are freed: in the suspend teardown (a good resume re-establishes it before __fbnic_open()) and on the resume open failure. A redundant ndo_stop() then walks an empty napi[]. The normal ndo_stop() down/up cycle is untouched and keeps num_napi for the next ndo_open(). Fixes: bc6107771bb4 ("eth: fbnic: Allocate a netdevice and napi vectors with queues") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942021809.7700.10804028989308077839.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/meta/fbnic/fbnic_pci.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 8b9bc9e8ea56..c6698e3002a1 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -434,6 +434,7 @@ static int fbnic_pm_suspend(struct device *dev) { struct fbnic_dev *fbd = dev_get_drvdata(dev); struct net_device *netdev = fbd->netdev; + struct fbnic_net *fbn; if (fbnic_init_failure(fbd)) goto null_uc_addr; @@ -441,11 +442,16 @@ static int fbnic_pm_suspend(struct device *dev) rtnl_lock(); netdev_lock(netdev); + fbn = netdev_priv(netdev); + netif_device_detach(netdev); if (netif_running(netdev)) netdev->netdev_ops->ndo_stop(netdev); + /* The IRQs are about to be freed, so drop the napi vector count */ + fbn->num_napi = 0; + netdev_unlock(netdev); rtnl_unlock(); @@ -508,16 +514,20 @@ static int __fbnic_pm_resume(struct device *dev) if (fbnic_init_failure(fbd)) return 0; + rtnl_lock(); + netdev_lock(netdev); + fbn = netdev_priv(netdev); /* Reset the queues if needed */ fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); - rtnl_lock(); - netdev_lock(netdev); - - if (netif_running(netdev)) + if (netif_running(netdev)) { err = __fbnic_open(fbn); + /* On failure the vectors are freed, so drop the count */ + if (err) + fbn->num_napi = 0; + } netdev_unlock(netdev); rtnl_unlock(); From 8947f13e436a4ff5eed9f8f019b2865a07af4bb2 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:10:25 -0700 Subject: [PATCH 5/6] eth: fbnic: Set AW_FLUSH_MODE alongside AW_FLUSH when flushing the mailbox When tearing down the FW mailbox Rx ring, fbnic_mbx_reset_desc_ring() writes AW_CFG with FLUSH set and everything else, BME included, cleared. Clearing BME halts the device's writes to the host but leaves the staged requests parked in the PUL write pipeline rather than draining them, so on the write path FLUSH alone never terminates the outstanding requests and the flush the firmware waits on never completes. Add the FLUSH_MODE definition and set both bits so the staged writes drain out of the pipeline on their own. BME stays cleared, so nothing lands on the host; it is restored later in fbnic_mbx_init_desc_ring() when the ring is rebuilt, once the outstanding writes are gone. The read path is unaffected. AR_CFG has no equivalent mode bit and AR_FLUSH terminates the outstanding reads by itself, so it is left as is. Both writes remain plain stores rather than read-modify-writes. That is deliberate: the matching write in fbnic_mbx_init_desc_ring() restores BME and the TLP attributes, and clears both flush bits as a side effect. Fixes: 3b12f00ddd08 ("fbnic: Gate AXI read/write enabling on FW mailbox") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942022583.7700.11050671998277309744.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/meta/fbnic/fbnic_csr.h | 1 + drivers/net/ethernet/meta/fbnic/fbnic_fw.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 64b958df7774..14af30e189d6 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -974,6 +974,7 @@ enum { /* PUL User Registers */ #define FBNIC_CSR_START_PUL_USER 0x31000 /* CSR section delimiter */ #define FBNIC_PUL_OB_TLP_HDR_AW_CFG 0x3103d /* 0xc40f4 */ +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE CSR_BIT(20) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH CSR_BIT(19) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME CSR_BIT(18) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR CSR_GENMASK(17, 15) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 283d25fae79e..59aa879798b9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -60,8 +60,15 @@ static void fbnic_mbx_reset_desc_ring(struct fbnic_dev *fbd, int mbx_idx) */ switch (mbx_idx) { case FBNIC_IPC_MBX_RX_IDX: + /* Clearing BME blocks the device from writing to the host + * but leaves the requests parked in the write pipeline. The + * write path only clears outstanding requests when both FLUSH + * and FLUSH_MODE are set; FLUSH_MODE lets them drain without + * landing on the host. + */ wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, - FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH); + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH | + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE); break; case FBNIC_IPC_MBX_TX_IDX: wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, From 1b97a269a5bdde20d4e69511f27649c9cb82b7c7 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:10:33 -0700 Subject: [PATCH 6/6] eth: fbnic: Handle FW mailbox completions flagged with an error The firmware can complete a mailbox descriptor while also setting FW_ERR to indicate it could not process the request, for example on a mailbox DMA error. The completion carries no valid data. The driver did not check FW_ERR. On the Rx mailbox it would sync and parse the stale page as a normal message, and on the Tx mailbox it silently freed the request. If the initial capabilities exchange in fbnic_mbx_poll_tx_ready() hit FW_ERR -- on the Tx request or on the Rx response descriptor -- no response was parsed and the poll spun until it timed out even though the ring was healthy. Check FW_ERR on both mailboxes. Count it per-mailbox in fbnic_fw_mbx.resp_error, which is also shown in debugfs, warn (rate limited, since the bit is firmware controlled), and drop the Rx page instead of parsing it. In fbnic_mbx_poll_tx_ready() re-issue the capabilities request when either the Tx or the Rx resp_error counter advances, so a FW_ERR on the request or on its response triggers a retry rather than a timeout. A valid capabilities response is honored before the retry check, so a response parsed in the same poll as an unrelated FW_ERR is not discarded. The counters are mailbox-wide rather than keyed to the capabilities request; that is sufficient here because the exchange runs during bring-up before any other mailbox traffic, and any spurious retry is bounded by the existing 10s timeout. Fixes: da3cde08209e ("eth: fbnic: Add FW communication mechanism") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942023343.7700.9423398932961964439.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/meta/fbnic/fbnic_csr.h | 4 ++ .../net/ethernet/meta/fbnic/fbnic_debugfs.c | 4 +- drivers/net/ethernet/meta/fbnic/fbnic_fw.c | 38 ++++++++++++++++++- drivers/net/ethernet/meta/fbnic/fbnic_fw.h | 1 + 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 14af30e189d6..baba3471bf5a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -1216,6 +1216,10 @@ enum { #define FBNIC_IPC_MBX_DESC_LEN_MASK DESC_GENMASK(63, 48) #define FBNIC_IPC_MBX_DESC_EOM DESC_BIT(46) #define FBNIC_IPC_MBX_DESC_ADDR_MASK DESC_GENMASK(45, 3) +/* Set with FW_CMPL when the FW completed a descriptor without successfully + * processing it (e.g. a mailbox DMA error); the completion has no valid data. + */ +#define FBNIC_IPC_MBX_DESC_FW_ERR DESC_BIT(2) #define FBNIC_IPC_MBX_DESC_FW_CMPL DESC_BIT(1) #define FBNIC_IPC_MBX_DESC_HOST_CMPL DESC_BIT(0) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index 3c4563c8f403..6edfa0aa69f1 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -539,8 +539,8 @@ static void fbnic_dbg_fw_mbx_display(struct seq_file *s, /* Generate header */ seq_puts(s, mbx_idx == FBNIC_IPC_MBX_RX_IDX ? "Rx\n" : "Tx\n"); - seq_printf(s, "Rdy: %d Head: %d Tail: %d\n", - mbx->ready, mbx->head, mbx->tail); + seq_printf(s, "Rdy: %d Head: %d Tail: %d resp_error: %llu\n", + mbx->ready, mbx->head, mbx->tail, mbx->resp_error); snprintf(hdr, sizeof(hdr), "%3s %-4s %s %-12s %s %-3s %-16s\n", "Idx", "Len", "E", "Addr", "F", "H", "Raw"); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 59aa879798b9..6d7eb8479edf 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -292,6 +292,12 @@ static void fbnic_mbx_process_tx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + tx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW completed a Tx mailbox request with an error\n"); + } + fbnic_mbx_unmap_and_free_msg(fbd, FBNIC_IPC_MBX_TX_IDX, head); head++; @@ -1673,6 +1679,13 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + rx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW reported an error on an Rx mailbox message; dropping\n"); + goto next_page; + } + dma_sync_single_for_cpu(fbd->dev, rx_mbx->buf_info[head].addr, FBNIC_RX_PAGE_SIZE, DMA_FROM_DEVICE); @@ -1740,7 +1753,9 @@ void fbnic_mbx_poll(struct fbnic_dev *fbd) int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) { struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX]; unsigned long timeout = jiffies + 10 * HZ + 1; + u64 tx_resp_error, rx_resp_error; int err, i; do { @@ -1771,6 +1786,9 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) * mgmt.version once we get the actual version from the firmware * in the capabilities request message. */ +send_cap_req: + tx_resp_error = tx_mbx->resp_error; + rx_resp_error = rx_mbx->resp_error; err = fbnic_fw_xmit_simple_msg(fbd, FBNIC_TLV_MSG_ID_HOST_CAP_REQ); if (err) goto clean_mbx; @@ -1788,9 +1806,27 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) msleep(20); fbnic_mbx_poll(fbd); + /* A valid capabilities response ends the poll. Check it + * before the FW_ERR retry below so a response parsed in the + * same poll as an unrelated FW_ERR is not discarded. + */ + if (fbd->fw_cap.running.mgmt.version >= MIN_FW_VER_CODE) + break; + /* set err, but wait till mgmt.version check to report it */ - if (!time_is_after_jiffies(timeout)) + if (!time_is_after_jiffies(timeout)) { err = -ETIMEDOUT; + continue; + } + + /* The FW can flag our capabilities request (Tx) or its + * response (Rx) with FW_ERR, in which case it produced no + * usable response. The ring is not wedged, so re-issue the + * request instead of spinning until the timeout. + */ + if (tx_mbx->resp_error != tx_resp_error || + rx_mbx->resp_error != rx_resp_error) + goto send_cap_req; } return 0; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index d84723e4cfa3..5f9969247e30 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -13,6 +13,7 @@ struct fbnic_tlv_msg; struct fbnic_fw_mbx { u8 ready, head, tail; + u64 resp_error; struct { struct fbnic_tlv_msg *msg; dma_addr_t addr;