mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch 'net-stmmac-xdp-path-fixes'
Lorenzo Bianconi says: ==================== net: stmmac: XDP path fixes This series adds some stmmac XDP preliminary fixes needed to introduce XDP RX multi-buff support. v1: https://lore.kernel.org/r/20261023-stmmac-xdp-fixes-v1-0-efb37e19f5bd@oss.qualcomm.com ==================== Link: https://patch.msgid.link/20261027-stmmac-xdp-fixes-v2-0-10eae8a093a8@oss.qualcomm.com Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni
commit
21f57ec673
|
|
@ -205,7 +205,7 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x,
|
|||
|
||||
if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) {
|
||||
x->rx_length++;
|
||||
return discard_frame;
|
||||
return discard_frame | rx_not_ls;
|
||||
}
|
||||
|
||||
if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) {
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ static int ndesc_get_rx_status(struct stmmac_extra_stats *x,
|
|||
|
||||
if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) {
|
||||
x->rx_length++;
|
||||
return discard_frame;
|
||||
return discard_frame | rx_not_ls;
|
||||
}
|
||||
|
||||
if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) {
|
||||
|
|
|
|||
|
|
@ -2182,6 +2182,9 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
|
|||
else
|
||||
dma_free_rx_skbufs(priv, dma_conf, queue);
|
||||
|
||||
dev_kfree_skb_any(rx_q->state.skb);
|
||||
rx_q->state.skb = NULL;
|
||||
rx_q->state_saved = false;
|
||||
rx_q->buf_alloc_num = 0;
|
||||
rx_q->xsk_pool = NULL;
|
||||
|
||||
|
|
@ -5099,22 +5102,35 @@ static netdev_features_t stmmac_features_check(struct sk_buff *skb,
|
|||
return vlan_features_check(skb, features);
|
||||
}
|
||||
|
||||
static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
|
||||
static int stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
|
||||
{
|
||||
struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb);
|
||||
__be16 vlan_proto = veth->h_vlan_proto;
|
||||
u16 vlanid;
|
||||
struct vlan_ethhdr *veth;
|
||||
|
||||
if ((vlan_proto == htons(ETH_P_8021Q) &&
|
||||
if (!pskb_may_pull(skb, ETH_HLEN))
|
||||
return -EINVAL;
|
||||
|
||||
veth = skb_vlan_eth_hdr(skb);
|
||||
if ((veth->h_vlan_proto == htons(ETH_P_8021Q) &&
|
||||
dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
|
||||
(vlan_proto == htons(ETH_P_8021AD) &&
|
||||
(veth->h_vlan_proto == htons(ETH_P_8021AD) &&
|
||||
dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
|
||||
__be16 vlan_proto = veth->h_vlan_proto;
|
||||
u8 meta_len = skb_metadata_len(skb);
|
||||
u16 vlanid;
|
||||
|
||||
/* pop the vlan tag */
|
||||
if (!pskb_may_pull(skb, VLAN_ETH_HLEN))
|
||||
return -EINVAL;
|
||||
|
||||
veth = skb_vlan_eth_hdr(skb);
|
||||
vlanid = ntohs(veth->h_vlan_TCI);
|
||||
memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
|
||||
memmove(skb->data - meta_len + VLAN_HLEN,
|
||||
skb->data - meta_len, meta_len + ETH_ALEN * 2);
|
||||
skb_pull(skb, VLAN_HLEN);
|
||||
__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool stmmac_rx_need_ic(struct stmmac_priv *priv,
|
||||
|
|
@ -5516,7 +5532,8 @@ static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch,
|
|||
return NULL;
|
||||
|
||||
skb_reserve(skb, xdp->data - xdp->data_hard_start);
|
||||
memcpy(__skb_put(skb, datasize), xdp->data, datasize);
|
||||
__skb_put(skb, datasize);
|
||||
memcpy(skb->data - metasize, xdp->data_meta, datasize + metasize);
|
||||
if (metasize)
|
||||
skb_metadata_set(skb, metasize);
|
||||
|
||||
|
|
@ -5542,12 +5559,15 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
|
|||
}
|
||||
|
||||
stmmac_get_rx_hwtstamp(priv, p, np, skb);
|
||||
if (priv->hw->hw_vlan_en)
|
||||
if (priv->hw->hw_vlan_en) {
|
||||
/* MAC level stripping. */
|
||||
stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
|
||||
else
|
||||
/* Driver level stripping. */
|
||||
stmmac_rx_vlan(priv->dev, skb);
|
||||
} else if (stmmac_rx_vlan(priv->dev, skb)) {
|
||||
dev_kfree_skb_any(skb);
|
||||
priv->xstats.rx_dropped++;
|
||||
return;
|
||||
}
|
||||
|
||||
skb->protocol = eth_type_trans(skb, priv->dev);
|
||||
|
||||
if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb))
|
||||
|
|
@ -5629,12 +5649,12 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
unsigned int count = 0, error = 0, len = 0;
|
||||
int dirty = stmmac_rx_dirty(priv, queue);
|
||||
unsigned int next_entry = rx_q->cur_rx;
|
||||
bool in_progress = rx_q->state_saved;
|
||||
u32 rx_errors = 0, rx_dropped = 0;
|
||||
unsigned int desc_size;
|
||||
struct bpf_prog *prog;
|
||||
bool failure = false;
|
||||
int xdp_status = 0;
|
||||
int status = 0;
|
||||
|
||||
if (netif_msg_rx_status(priv)) {
|
||||
void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0);
|
||||
|
|
@ -5645,23 +5665,25 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
|
||||
rx_q->dma_rx_phy, desc_size);
|
||||
}
|
||||
|
||||
if (rx_q->state_saved) {
|
||||
error = rx_q->state.error;
|
||||
len = rx_q->state.len;
|
||||
rx_q->state_saved = false;
|
||||
}
|
||||
|
||||
while (count < limit) {
|
||||
struct stmmac_rx_buffer *buf;
|
||||
struct stmmac_xdp_buff *ctx;
|
||||
unsigned int buf1_len = 0;
|
||||
struct dma_desc *np, *p;
|
||||
int entry;
|
||||
int entry, status;
|
||||
int res;
|
||||
|
||||
if (!count && rx_q->state_saved) {
|
||||
error = rx_q->state.error;
|
||||
len = rx_q->state.len;
|
||||
} else {
|
||||
rx_q->state_saved = false;
|
||||
if (!in_progress) {
|
||||
error = 0;
|
||||
len = 0;
|
||||
}
|
||||
|
||||
read_again:
|
||||
if (count >= limit)
|
||||
break;
|
||||
|
|
@ -5700,6 +5722,8 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
if (!buf->xdp)
|
||||
break;
|
||||
|
||||
in_progress = status & rx_not_ls;
|
||||
|
||||
if (priv->extend_desc) {
|
||||
/* Read the extended status after the own bit */
|
||||
dma_rmb();
|
||||
|
|
@ -5707,18 +5731,21 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
&rx_q->dma_erx[entry],
|
||||
status);
|
||||
}
|
||||
if (unlikely(status == discard_frame)) {
|
||||
xsk_buff_free(buf->xdp);
|
||||
buf->xdp = NULL;
|
||||
dirty++;
|
||||
|
||||
if (unlikely(!error && (status & discard_frame))) {
|
||||
error = 1;
|
||||
if (!priv->hwts_rx_en)
|
||||
rx_errors++;
|
||||
}
|
||||
|
||||
if (unlikely(error && (status & rx_not_ls)))
|
||||
goto read_again;
|
||||
if (unlikely(error)) {
|
||||
xsk_buff_free(buf->xdp);
|
||||
buf->xdp = NULL;
|
||||
dirty++;
|
||||
|
||||
if (status & rx_not_ls)
|
||||
goto read_again;
|
||||
|
||||
count++;
|
||||
continue;
|
||||
}
|
||||
|
|
@ -5727,8 +5754,9 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
if (likely(status & rx_not_ls)) {
|
||||
xsk_buff_free(buf->xdp);
|
||||
buf->xdp = NULL;
|
||||
rx_dropped++;
|
||||
error = 1;
|
||||
dirty++;
|
||||
count++;
|
||||
goto read_again;
|
||||
}
|
||||
|
||||
|
|
@ -5776,7 +5804,7 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
count++;
|
||||
}
|
||||
|
||||
if (status & rx_not_ls) {
|
||||
if (in_progress) {
|
||||
rx_q->state_saved = true;
|
||||
rx_q->state.error = error;
|
||||
rx_q->state.len = len;
|
||||
|
|
@ -5818,9 +5846,10 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
|
||||
struct stmmac_channel *ch = &priv->channel[queue];
|
||||
unsigned int count = 0, error = 0, len = 0;
|
||||
int status = 0, coe = priv->hw->rx_csum;
|
||||
unsigned int next_entry = rx_q->cur_rx;
|
||||
bool in_progress = rx_q->state_saved;
|
||||
enum dma_data_direction dma_dir;
|
||||
int coe = priv->hw->rx_csum;
|
||||
unsigned int desc_size;
|
||||
struct sk_buff *skb = NULL;
|
||||
struct stmmac_xdp_buff ctx;
|
||||
|
|
@ -5840,25 +5869,28 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
|
||||
rx_q->dma_rx_phy, desc_size);
|
||||
}
|
||||
|
||||
if (rx_q->state_saved) {
|
||||
skb = rx_q->state.skb;
|
||||
error = rx_q->state.error;
|
||||
len = rx_q->state.len;
|
||||
rx_q->state.skb = NULL;
|
||||
rx_q->state_saved = false;
|
||||
}
|
||||
|
||||
while (count < limit) {
|
||||
unsigned int buf1_len = 0, buf2_len = 0;
|
||||
enum pkt_hash_types hash_type;
|
||||
struct stmmac_rx_buffer *buf;
|
||||
struct dma_desc *np, *p;
|
||||
int entry;
|
||||
int entry, status;
|
||||
u32 hash;
|
||||
|
||||
if (!count && rx_q->state_saved) {
|
||||
skb = rx_q->state.skb;
|
||||
error = rx_q->state.error;
|
||||
len = rx_q->state.len;
|
||||
} else {
|
||||
rx_q->state_saved = false;
|
||||
if (!in_progress) {
|
||||
skb = NULL;
|
||||
error = 0;
|
||||
len = 0;
|
||||
}
|
||||
|
||||
read_again:
|
||||
if (count >= limit)
|
||||
break;
|
||||
|
|
@ -5887,6 +5919,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
|
||||
prefetch(np);
|
||||
|
||||
in_progress = status & rx_not_ls;
|
||||
|
||||
if (priv->extend_desc) {
|
||||
/* Read the extended status after the own bit */
|
||||
dma_rmb();
|
||||
|
|
@ -5894,21 +5928,26 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
&rx_q->dma_erx[entry],
|
||||
status);
|
||||
}
|
||||
if (unlikely(status == discard_frame)) {
|
||||
page_pool_put_page(rx_q->page_pool, buf->page, 0, true);
|
||||
buf->page = NULL;
|
||||
|
||||
if (unlikely(!error && (status & discard_frame))) {
|
||||
error = 1;
|
||||
if (!priv->hwts_rx_en)
|
||||
rx_errors++;
|
||||
}
|
||||
|
||||
if (unlikely(error && (status & rx_not_ls)))
|
||||
goto read_again;
|
||||
/* We do not support SG in XDP mode */
|
||||
if (unlikely(!error && (status & rx_not_ls) &&
|
||||
stmmac_xdp_is_enabled(priv))) {
|
||||
rx_dropped++;
|
||||
error = 1;
|
||||
}
|
||||
|
||||
if (unlikely(error)) {
|
||||
if (status & rx_not_ls)
|
||||
goto read_again;
|
||||
|
||||
dev_kfree_skb(skb);
|
||||
skb = NULL;
|
||||
count++;
|
||||
continue;
|
||||
goto next;
|
||||
}
|
||||
|
||||
/* Buffer is good. Go on. */
|
||||
|
|
@ -5918,13 +5957,19 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
|
||||
len += buf2_len;
|
||||
|
||||
/* Drop the packet if it is split over buf1 and buf2 on a
|
||||
* single descriptor if the interface is running in XDP mode.
|
||||
*/
|
||||
if (unlikely(stmmac_xdp_is_enabled(priv) && buf2_len)) {
|
||||
rx_dropped++;
|
||||
goto next;
|
||||
}
|
||||
|
||||
/* ACS is disabled; strip manually. */
|
||||
if (likely(!(status & rx_not_ls)))
|
||||
len -= ETH_FCS_LEN;
|
||||
|
||||
if (!skb) {
|
||||
unsigned int pre_len, sync_len;
|
||||
|
||||
/* Each frame starts here: reset the FCS handling */
|
||||
fcs_stripped = false;
|
||||
|
||||
|
|
@ -5942,49 +5987,26 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
xdp_prepare_buff(&ctx.xdp, page_address(buf->page),
|
||||
buf->page_offset, buf1_len, true);
|
||||
|
||||
pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
|
||||
buf->page_offset;
|
||||
|
||||
ctx.priv = priv;
|
||||
ctx.desc = p;
|
||||
ctx.ndesc = np;
|
||||
|
||||
skb = stmmac_xdp_run_prog(priv, &ctx.xdp);
|
||||
/* Due xdp_adjust_tail: DMA sync for_device
|
||||
* cover max len CPU touch
|
||||
*/
|
||||
sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
|
||||
buf->page_offset;
|
||||
sync_len = max(sync_len, pre_len);
|
||||
|
||||
/* For Not XDP_PASS verdict */
|
||||
if (IS_ERR(skb)) {
|
||||
unsigned int xdp_res = -PTR_ERR(skb);
|
||||
|
||||
if (xdp_res & STMMAC_XDP_CONSUMED) {
|
||||
page_pool_put_page(rx_q->page_pool,
|
||||
virt_to_head_page(ctx.xdp.data),
|
||||
sync_len, true);
|
||||
page_pool_recycle_direct(rx_q->page_pool,
|
||||
virt_to_head_page(ctx.xdp.data));
|
||||
buf->page = NULL;
|
||||
rx_dropped++;
|
||||
|
||||
/* Clear skb as it was set as
|
||||
* status by XDP program.
|
||||
*/
|
||||
skb = NULL;
|
||||
|
||||
if (unlikely((status & rx_not_ls)))
|
||||
goto read_again;
|
||||
|
||||
count++;
|
||||
continue;
|
||||
goto next;
|
||||
} else if (xdp_res & (STMMAC_XDP_TX |
|
||||
STMMAC_XDP_REDIRECT)) {
|
||||
xdp_status |= xdp_res;
|
||||
buf->page = NULL;
|
||||
skb = NULL;
|
||||
count++;
|
||||
continue;
|
||||
goto next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -6041,20 +6063,20 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
/* Remove FCS if needed */
|
||||
if (!fcs_stripped && pskb_trim(skb, len)) {
|
||||
dev_kfree_skb_any(skb);
|
||||
skb = NULL;
|
||||
rx_dropped++;
|
||||
count++;
|
||||
continue;
|
||||
goto next;
|
||||
}
|
||||
|
||||
stmmac_get_rx_hwtstamp(priv, p, np, skb);
|
||||
|
||||
if (priv->hw->hw_vlan_en)
|
||||
if (priv->hw->hw_vlan_en) {
|
||||
/* MAC level stripping. */
|
||||
stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
|
||||
else
|
||||
/* Driver level stripping. */
|
||||
stmmac_rx_vlan(priv->dev, skb);
|
||||
} else if (stmmac_rx_vlan(priv->dev, skb)) {
|
||||
dev_kfree_skb_any(skb);
|
||||
rx_dropped++;
|
||||
goto next;
|
||||
}
|
||||
|
||||
skb->protocol = eth_type_trans(skb, priv->dev);
|
||||
|
||||
|
|
@ -6069,14 +6091,15 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
|
|||
|
||||
skb_record_rx_queue(skb, queue);
|
||||
napi_gro_receive(&ch->rx_napi, skb);
|
||||
skb = NULL;
|
||||
|
||||
rx_packets++;
|
||||
rx_bytes += len;
|
||||
next:
|
||||
in_progress = false;
|
||||
skb = NULL;
|
||||
count++;
|
||||
}
|
||||
|
||||
if (status & rx_not_ls || skb) {
|
||||
if (in_progress || skb) {
|
||||
rx_q->state_saved = true;
|
||||
rx_q->state.skb = skb;
|
||||
rx_q->state.error = error;
|
||||
|
|
@ -8421,6 +8444,9 @@ static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)
|
|||
{
|
||||
struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
|
||||
|
||||
dev_kfree_skb_any(rx_q->state.skb);
|
||||
rx_q->state.skb = NULL;
|
||||
rx_q->state_saved = false;
|
||||
rx_q->cur_rx = 0;
|
||||
rx_q->dirty_rx = 0;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user