diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c index 758554c951c5..949994874ed9 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ +#include #include #include @@ -139,7 +140,123 @@ static bool lan966x_fdma_pci_rx_size_fits(struct fdma *fdma, u32 blockl) blockl <= fdma->db_size - XDP_PACKET_HEADROOM; } -static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) +static int lan966x_fdma_pci_xmit_xdpf(struct lan966x_port *port, + void *ptr, u32 len) +{ + struct lan966x *lan966x = port->lan966x; + struct lan966x_tx *tx = &lan966x->tx; + struct fdma *fdma = &tx->fdma; + int next_to_use, ret = 0; + void *virt_addr; + + spin_lock(&lan966x->tx_lock); + + next_to_use = lan966x_fdma_pci_get_next_dcb(fdma); + + if (next_to_use < 0) { + netif_stop_queue(port->dev); + port->dev->stats.tx_dropped++; + ret = NETDEV_TX_BUSY; + goto out; + } + + /* Only the upper bound is enforced: XDP owns the frame contents and + * length, so a program that shrinks below ETH_ZLEN gets what it asked + * for. + */ + if (!lan966x_fdma_pci_tx_size_fits(fdma, len)) { + port->dev->stats.tx_dropped++; + ret = -EINVAL; + goto out; + } + + /* virt_addr points to the IFH. */ + virt_addr = fdma_dataptr_virt_addr_contiguous(fdma, next_to_use, 0); + + /* Construct a fresh IFH. */ + memset(virt_addr, 0, IFH_LEN_BYTES); + lan966x_ifh_set_bypass(virt_addr, 1); + lan966x_ifh_set_port(virt_addr, BIT_ULL(port->chip_port)); + + /* Copy the (post-XDP) frame after the IFH. */ + memcpy(virt_addr + IFH_LEN_BYTES, ptr, len); + + /* Order frame write before DCB status write below. */ + dma_wmb(); + + /* Reserve ETH_FCS_LEN for the HW-inserted FCS (len is FCS-stripped). */ + fdma_dcb_add(fdma, + next_to_use, + 0, + FDMA_DCB_STATUS_INTR | + FDMA_DCB_STATUS_SOF | + FDMA_DCB_STATUS_EOF | + FDMA_DCB_STATUS_BLOCKO(0) | + FDMA_DCB_STATUS_BLOCKL(IFH_LEN_BYTES + len + ETH_FCS_LEN)); + + /* Start the transmission. */ + lan966x_fdma_tx_start(tx); + + port->dev->stats.tx_bytes += len; + port->dev->stats.tx_packets++; + +out: + spin_unlock(&lan966x->tx_lock); + + return ret; +} + +static int lan966x_xdp_pci_run(struct lan966x_port *port, void *data, + u32 data_len, void **xdp_data, u32 *xdp_len) +{ + /* Read once so the NULL check and bpf_prog_run_xdp() see the same + * pointer. + */ + struct bpf_prog *xdp_prog = READ_ONCE(port->xdp_prog); + struct lan966x *lan966x = port->lan966x; + struct fdma *fdma = &lan966x->rx.fdma; + struct xdp_buff xdp; + u32 act; + + if (!xdp_prog) + return FDMA_PASS; + + xdp_init_buff(&xdp, fdma->db_size, &port->xdp_rxq); + + /* hard_start is set to slot start (virt_addr is XDP_PACKET_HEADROOM + * into the slot). Headroom includes the IFH; BPF may grow into it + * via adjust_head. IFH is rebuilt on XDP_TX and unread on XDP_PASS. + */ + xdp_prepare_buff(&xdp, + data - XDP_PACKET_HEADROOM, + XDP_PACKET_HEADROOM + IFH_LEN_BYTES, + data_len, + false); + + act = bpf_prog_run_xdp(xdp_prog, &xdp); + + *xdp_data = xdp.data; + *xdp_len = xdp.data_end - xdp.data; + + switch (act) { + case XDP_PASS: + return FDMA_PASS; + case XDP_TX: + return lan966x_fdma_pci_xmit_xdpf(port, *xdp_data, *xdp_len) ? + FDMA_DROP : FDMA_TX; + default: + bpf_warn_invalid_xdp_action(port->dev, xdp_prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(port->dev, xdp_prog, act); + fallthrough; + case XDP_DROP: + return FDMA_DROP; + } +} + +static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port, + void **data, u32 *data_len) { struct lan966x *lan966x = rx->lan966x; struct fdma *fdma = &rx->fdma; @@ -171,7 +288,15 @@ static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) if (!lan966x_fdma_pci_rx_size_fits(fdma, blockl)) return FDMA_ERROR; - return FDMA_PASS; + /* Present the Ethernet frame (no IFH, no FCS). HW re-inserts the + * FCS on TX; see lan966x_fdma_pci_xmit_xdpf(). May be overridden + * by XDP. The FCS strip is unconditional because NETIF_F_RXFCS + * is not advertised in hw_features. + */ + *data = virt_addr + IFH_LEN_BYTES; + *data_len = blockl - IFH_LEN_BYTES - ETH_FCS_LEN; + + return lan966x_xdp_pci_run(port, virt_addr, *data_len, data, data_len); } static bool lan966x_fdma_pci_hw_offload(struct lan966x *lan966x, u32 port, @@ -206,34 +331,21 @@ static bool lan966x_fdma_pci_hw_offload(struct lan966x *lan966x, u32 port, } static struct sk_buff *lan966x_fdma_pci_rx_get_frame(struct lan966x_rx *rx, - u64 src_port) + u64 src_port, void *data, + u32 data_len) { struct lan966x *lan966x = rx->lan966x; - struct fdma *fdma = &rx->fdma; struct sk_buff *skb; - struct fdma_db *db; - u32 data_len; - - /* Get the received frame and create an SKB for it. */ - db = fdma_db_next_get(fdma); - data_len = fdma_db_len_get(db); skb = napi_alloc_skb(&lan966x->napi, data_len); if (unlikely(!skb)) return NULL; - memcpy(skb->data, - fdma_dataptr_virt_addr_contiguous(fdma, - fdma->dcb_index, - fdma->db_index), - data_len); + memcpy(skb->data, data, data_len); skb_put(skb, data_len); skb->dev = lan966x->ports[src_port]->dev; - skb_pull(skb, IFH_LEN_BYTES); - - skb_trim(skb, skb->len - ETH_FCS_LEN); skb->protocol = eth_type_trans(skb, skb->dev); @@ -324,6 +436,8 @@ static int lan966x_fdma_pci_napi_poll(struct napi_struct *napi, int weight) struct sk_buff *skb; int counter = 0; u64 src_port; + u32 data_len; + void *data; /* Wake any stopped TX queues if a TX DCB is available. */ spin_lock(&lan966x->tx_lock); @@ -340,7 +454,10 @@ static int lan966x_fdma_pci_napi_poll(struct napi_struct *napi, int weight) /* Order DONE read before DCB/frame reads below. */ dma_rmb(); counter++; - switch (lan966x_fdma_pci_rx_check_frame(rx, &src_port)) { + switch (lan966x_fdma_pci_rx_check_frame(rx, + &src_port, + &data, + &data_len)) { case FDMA_PASS: break; case FDMA_ERROR: @@ -349,8 +466,17 @@ static int lan966x_fdma_pci_napi_poll(struct napi_struct *napi, int weight) */ fdma_dcb_advance(fdma); continue; + case FDMA_TX: + fdma_dcb_advance(fdma); + continue; + case FDMA_DROP: + fdma_dcb_advance(fdma); + continue; } - skb = lan966x_fdma_pci_rx_get_frame(rx, src_port); + skb = lan966x_fdma_pci_rx_get_frame(rx, + src_port, + data, + data_len); fdma_dcb_advance(fdma); if (!skb) { lan966x->ports[src_port]->dev->stats.rx_dropped++; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index c3afc4cc597f..e4c9c5812cca 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -875,11 +875,13 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, port->phylink = phylink; - /* XDP is not supported on the PCIe FDMA path. */ - if (lan966x->fdma && !lan966x_is_pci(lan966x)) - dev->xdp_features = NETDEV_XDP_ACT_BASIC | - NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_NDO_XMIT; + if (lan966x->fdma) { + dev->xdp_features = NETDEV_XDP_ACT_BASIC; + + if (!lan966x_is_pci(lan966x)) + dev->xdp_features |= NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT; + } err = register_netdev(dev); if (err) { diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c index e63634887f64..b98426afc785 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c @@ -20,16 +20,16 @@ static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) return -EOPNOTSUPP; } - if (lan966x_is_pci(lan966x)) { - NL_SET_ERR_MSG_MOD(xdp->extack, - "XDP is not supported on the PCIe FDMA path"); - return -EOPNOTSUPP; - } - old_xdp = lan966x_xdp_present(lan966x); old_prog = xchg(&port->xdp_prog, xdp->prog); new_xdp = lan966x_xdp_present(lan966x); + /* PCIe FDMA uses contiguous buffers, so no page_pool reload + * is needed. + */ + if (lan966x_is_pci(lan966x)) + goto out; + if (old_xdp == new_xdp) goto out;