diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c index 023b54b72478..7ece8d4157a8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c @@ -205,7 +205,7 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x, if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) { x->rx_length++; - return discard_frame; + return discard_frame | rx_not_ls; } if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) { diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c index c4b613564f87..d19bf7ca4db3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c @@ -70,7 +70,7 @@ static int ndesc_get_rx_status(struct stmmac_extra_stats *x, if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) { x->rx_length++; - return discard_frame; + return discard_frame | rx_not_ls; } if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 9ded31150fbe..b038472b9817 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2182,6 +2182,9 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, else dma_free_rx_skbufs(priv, dma_conf, queue); + dev_kfree_skb_any(rx_q->state.skb); + rx_q->state.skb = NULL; + rx_q->state_saved = false; rx_q->buf_alloc_num = 0; rx_q->xsk_pool = NULL; @@ -5099,22 +5102,35 @@ static netdev_features_t stmmac_features_check(struct sk_buff *skb, return vlan_features_check(skb, features); } -static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) +static int stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) { - struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); - __be16 vlan_proto = veth->h_vlan_proto; - u16 vlanid; + struct vlan_ethhdr *veth; - if ((vlan_proto == htons(ETH_P_8021Q) && + if (!pskb_may_pull(skb, ETH_HLEN)) + return -EINVAL; + + veth = skb_vlan_eth_hdr(skb); + if ((veth->h_vlan_proto == htons(ETH_P_8021Q) && dev->features & NETIF_F_HW_VLAN_CTAG_RX) || - (vlan_proto == htons(ETH_P_8021AD) && + (veth->h_vlan_proto == htons(ETH_P_8021AD) && dev->features & NETIF_F_HW_VLAN_STAG_RX)) { + __be16 vlan_proto = veth->h_vlan_proto; + u8 meta_len = skb_metadata_len(skb); + u16 vlanid; + /* pop the vlan tag */ + if (!pskb_may_pull(skb, VLAN_ETH_HLEN)) + return -EINVAL; + + veth = skb_vlan_eth_hdr(skb); vlanid = ntohs(veth->h_vlan_TCI); - memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); + memmove(skb->data - meta_len + VLAN_HLEN, + skb->data - meta_len, meta_len + ETH_ALEN * 2); skb_pull(skb, VLAN_HLEN); __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); } + + return 0; } static bool stmmac_rx_need_ic(struct stmmac_priv *priv, @@ -5516,7 +5532,8 @@ static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, return NULL; skb_reserve(skb, xdp->data - xdp->data_hard_start); - memcpy(__skb_put(skb, datasize), xdp->data, datasize); + __skb_put(skb, datasize); + memcpy(skb->data - metasize, xdp->data_meta, datasize + metasize); if (metasize) skb_metadata_set(skb, metasize); @@ -5542,12 +5559,15 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, } stmmac_get_rx_hwtstamp(priv, p, np, skb); - if (priv->hw->hw_vlan_en) + if (priv->hw->hw_vlan_en) { /* MAC level stripping. */ stmmac_rx_hw_vlan(priv, priv->hw, p, skb); - else - /* Driver level stripping. */ - stmmac_rx_vlan(priv->dev, skb); + } else if (stmmac_rx_vlan(priv->dev, skb)) { + dev_kfree_skb_any(skb); + priv->xstats.rx_dropped++; + return; + } + skb->protocol = eth_type_trans(skb, priv->dev); if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) @@ -5629,12 +5649,12 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) unsigned int count = 0, error = 0, len = 0; int dirty = stmmac_rx_dirty(priv, queue); unsigned int next_entry = rx_q->cur_rx; + bool in_progress = rx_q->state_saved; u32 rx_errors = 0, rx_dropped = 0; unsigned int desc_size; struct bpf_prog *prog; bool failure = false; int xdp_status = 0; - int status = 0; if (netif_msg_rx_status(priv)) { void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); @@ -5645,23 +5665,25 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, rx_q->dma_rx_phy, desc_size); } + + if (rx_q->state_saved) { + error = rx_q->state.error; + len = rx_q->state.len; + rx_q->state_saved = false; + } + while (count < limit) { struct stmmac_rx_buffer *buf; struct stmmac_xdp_buff *ctx; unsigned int buf1_len = 0; struct dma_desc *np, *p; - int entry; + int entry, status; int res; - if (!count && rx_q->state_saved) { - error = rx_q->state.error; - len = rx_q->state.len; - } else { - rx_q->state_saved = false; + if (!in_progress) { error = 0; len = 0; } - read_again: if (count >= limit) break; @@ -5700,6 +5722,8 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) if (!buf->xdp) break; + in_progress = status & rx_not_ls; + if (priv->extend_desc) { /* Read the extended status after the own bit */ dma_rmb(); @@ -5707,18 +5731,21 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) &rx_q->dma_erx[entry], status); } - if (unlikely(status == discard_frame)) { - xsk_buff_free(buf->xdp); - buf->xdp = NULL; - dirty++; + + if (unlikely(!error && (status & discard_frame))) { error = 1; if (!priv->hwts_rx_en) rx_errors++; } - if (unlikely(error && (status & rx_not_ls))) - goto read_again; if (unlikely(error)) { + xsk_buff_free(buf->xdp); + buf->xdp = NULL; + dirty++; + + if (status & rx_not_ls) + goto read_again; + count++; continue; } @@ -5727,8 +5754,9 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) if (likely(status & rx_not_ls)) { xsk_buff_free(buf->xdp); buf->xdp = NULL; + rx_dropped++; + error = 1; dirty++; - count++; goto read_again; } @@ -5776,7 +5804,7 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) count++; } - if (status & rx_not_ls) { + if (in_progress) { rx_q->state_saved = true; rx_q->state.error = error; rx_q->state.len = len; @@ -5818,9 +5846,10 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; unsigned int count = 0, error = 0, len = 0; - int status = 0, coe = priv->hw->rx_csum; unsigned int next_entry = rx_q->cur_rx; + bool in_progress = rx_q->state_saved; enum dma_data_direction dma_dir; + int coe = priv->hw->rx_csum; unsigned int desc_size; struct sk_buff *skb = NULL; struct stmmac_xdp_buff ctx; @@ -5840,25 +5869,28 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, rx_q->dma_rx_phy, desc_size); } + + if (rx_q->state_saved) { + skb = rx_q->state.skb; + error = rx_q->state.error; + len = rx_q->state.len; + rx_q->state.skb = NULL; + rx_q->state_saved = false; + } + while (count < limit) { unsigned int buf1_len = 0, buf2_len = 0; enum pkt_hash_types hash_type; struct stmmac_rx_buffer *buf; struct dma_desc *np, *p; - int entry; + int entry, status; u32 hash; - if (!count && rx_q->state_saved) { - skb = rx_q->state.skb; - error = rx_q->state.error; - len = rx_q->state.len; - } else { - rx_q->state_saved = false; + if (!in_progress) { skb = NULL; error = 0; len = 0; } - read_again: if (count >= limit) break; @@ -5887,6 +5919,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) prefetch(np); + in_progress = status & rx_not_ls; + if (priv->extend_desc) { /* Read the extended status after the own bit */ dma_rmb(); @@ -5894,21 +5928,26 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) &rx_q->dma_erx[entry], status); } - if (unlikely(status == discard_frame)) { - page_pool_put_page(rx_q->page_pool, buf->page, 0, true); - buf->page = NULL; + + if (unlikely(!error && (status & discard_frame))) { error = 1; if (!priv->hwts_rx_en) rx_errors++; } - if (unlikely(error && (status & rx_not_ls))) - goto read_again; + /* We do not support SG in XDP mode */ + if (unlikely(!error && (status & rx_not_ls) && + stmmac_xdp_is_enabled(priv))) { + rx_dropped++; + error = 1; + } + if (unlikely(error)) { + if (status & rx_not_ls) + goto read_again; + dev_kfree_skb(skb); - skb = NULL; - count++; - continue; + goto next; } /* Buffer is good. Go on. */ @@ -5918,13 +5957,19 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) buf2_len = stmmac_rx_buf2_len(priv, p, status, len); len += buf2_len; + /* Drop the packet if it is split over buf1 and buf2 on a + * single descriptor if the interface is running in XDP mode. + */ + if (unlikely(stmmac_xdp_is_enabled(priv) && buf2_len)) { + rx_dropped++; + goto next; + } + /* ACS is disabled; strip manually. */ if (likely(!(status & rx_not_ls))) len -= ETH_FCS_LEN; if (!skb) { - unsigned int pre_len, sync_len; - /* Each frame starts here: reset the FCS handling */ fcs_stripped = false; @@ -5942,49 +5987,26 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) xdp_prepare_buff(&ctx.xdp, page_address(buf->page), buf->page_offset, buf1_len, true); - pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - - buf->page_offset; - ctx.priv = priv; ctx.desc = p; ctx.ndesc = np; skb = stmmac_xdp_run_prog(priv, &ctx.xdp); - /* Due xdp_adjust_tail: DMA sync for_device - * cover max len CPU touch - */ - sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - - buf->page_offset; - sync_len = max(sync_len, pre_len); - /* For Not XDP_PASS verdict */ if (IS_ERR(skb)) { unsigned int xdp_res = -PTR_ERR(skb); if (xdp_res & STMMAC_XDP_CONSUMED) { - page_pool_put_page(rx_q->page_pool, - virt_to_head_page(ctx.xdp.data), - sync_len, true); + page_pool_recycle_direct(rx_q->page_pool, + virt_to_head_page(ctx.xdp.data)); buf->page = NULL; rx_dropped++; - - /* Clear skb as it was set as - * status by XDP program. - */ - skb = NULL; - - if (unlikely((status & rx_not_ls))) - goto read_again; - - count++; - continue; + goto next; } else if (xdp_res & (STMMAC_XDP_TX | STMMAC_XDP_REDIRECT)) { xdp_status |= xdp_res; buf->page = NULL; - skb = NULL; - count++; - continue; + goto next; } } } @@ -6041,20 +6063,20 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) /* Remove FCS if needed */ if (!fcs_stripped && pskb_trim(skb, len)) { dev_kfree_skb_any(skb); - skb = NULL; rx_dropped++; - count++; - continue; + goto next; } stmmac_get_rx_hwtstamp(priv, p, np, skb); - if (priv->hw->hw_vlan_en) + if (priv->hw->hw_vlan_en) { /* MAC level stripping. */ stmmac_rx_hw_vlan(priv, priv->hw, p, skb); - else - /* Driver level stripping. */ - stmmac_rx_vlan(priv->dev, skb); + } else if (stmmac_rx_vlan(priv->dev, skb)) { + dev_kfree_skb_any(skb); + rx_dropped++; + goto next; + } skb->protocol = eth_type_trans(skb, priv->dev); @@ -6069,14 +6091,15 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) skb_record_rx_queue(skb, queue); napi_gro_receive(&ch->rx_napi, skb); - skb = NULL; - rx_packets++; rx_bytes += len; +next: + in_progress = false; + skb = NULL; count++; } - if (status & rx_not_ls || skb) { + if (in_progress || skb) { rx_q->state_saved = true; rx_q->state.skb = skb; rx_q->state.error = error; @@ -8421,6 +8444,9 @@ static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) { struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + dev_kfree_skb_any(rx_q->state.skb); + rx_q->state.skb = NULL; + rx_q->state_saved = false; rx_q->cur_rx = 0; rx_q->dirty_rx = 0; }