diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index b3883a5a7d7a..16fcc8cf6191 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -3082,7 +3082,7 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring) meta_req.tx_buffer = bi; meta_req.meta = meta; meta_req.used_desc = 0; - xsk_tx_metadata_request(meta, &igc_xsk_tx_metadata_ops, + xsk_tx_metadata_request(pool, &meta, &igc_xsk_tx_metadata_ops, &meta_req); /* xsk_tx_metadata_request() may have updated next_to_use */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 6867a5aed42c..822859c360e7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -483,7 +483,7 @@ typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *); typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *, struct mlx5e_xmit_data *, int, - struct xsk_tx_metadata *); + struct xsk_tx_metadata **); struct mlx5e_xdpsq { /* data path */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index d8c7cb8837d7..77ea51bfbaae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -452,11 +452,11 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta); + int check_result, struct xsk_tx_metadata **meta); INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta) + int check_result, struct xsk_tx_metadata **meta) { struct mlx5e_tx_mpwqe *session = &sq->mpwqe; struct mlx5e_xdpsq_stats *stats = sq->stats; @@ -504,7 +504,10 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx * and it's safe to complete it at any time. */ mlx5e_xdp_mpwqe_session_start(sq); - xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, &session->wqe->eth); + if (meta) + xsk_tx_metadata_request(sq->xsk_pool, meta, + &mlx5e_xsk_tx_metadata_ops, + &session->wqe->eth); } mlx5e_xdp_mpwqe_add_dseg(sq, p, stats); @@ -535,7 +538,7 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta) + int check_result, struct xsk_tx_metadata **meta) { struct mlx5e_xmit_data_frags *xdptxdf = container_of(xdptxd, struct mlx5e_xmit_data_frags, xd); @@ -649,7 +652,9 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, sq->pc += num_wqebbs; - xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, eseg); + if (meta) + xsk_tx_metadata_request(sq->xsk_pool, meta, + &mlx5e_xsk_tx_metadata_ops, eseg); sq->doorbell_cseg = cseg; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index 3c54f8962664..5b8f4094d553 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -114,11 +114,11 @@ extern const struct xsk_tx_metadata_ops mlx5e_xsk_tx_metadata_ops; INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, int check_result, - struct xsk_tx_metadata *meta)); + struct xsk_tx_metadata **meta)); INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, int check_result, - struct xsk_tx_metadata *meta)); + struct xsk_tx_metadata **meta)); INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)); INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 8aeab4b21035..3d19dad8f868 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -105,7 +105,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, mlx5e_xmit_xdp_frame, sq, &xdptxd, - check_result, meta); + check_result, &meta); if (unlikely(!ret)) { if (sq->mpwqe.wqe) mlx5e_xdp_mpwqe_complete(sq); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 3801f9d45278..97b28cfa14c2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2748,8 +2748,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) meta_req.set_ic = &set_ic; meta_req.tbs = tx_q->tbs; meta_req.edesc = &tx_q->dma_entx[entry]; - xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, - &meta_req); + xsk_tx_metadata_request(pool, &meta, + &stmmac_xsk_tx_metadata_ops, &meta_req); if (set_ic) { tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, tx_desc); diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 82b5d21aae87..5dcc0d7f65b7 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -205,7 +205,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, BUILD_BUG_ON(!__builtin_constant_p(tmo == libeth_xsktmo)); tmo = tmo == libeth_xsktmo ? &__libeth_xsktmo : tmo; - xsk_tx_metadata_request(ctx.meta, tmo, &desc); + xsk_tx_metadata_request(sq->pool, &ctx.meta, tmo, &desc); return desc; } diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 8b51876efbed..6e70b320b399 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -141,45 +141,16 @@ INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *)); static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, struct xsk_tx_metadata_compl *compl) { + compl->tx_timestamp = NULL; + if (!meta) return; - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - compl->tx_timestamp = &meta->completion.tx_timestamp; - else - compl->tx_timestamp = NULL; -} + /* we can only arrive here if the completion timestamp has been + * requested via XDP_TXMD_FLAGS_TIMESTAMP, see xsk_tx_metadata_request + */ -/** - * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission - * and call appropriate xsk_tx_metadata_ops operation. - * @meta: pointer to AF_XDP metadata area - * @ops: pointer to struct xsk_tx_metadata_ops - * @priv: pointer to driver-private aread - * - * This function should be called by the networking device when - * it prepares AF_XDP egress packet. - */ -static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ - if (!meta) - return; - - if (ops->tmo_request_launch_time) - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - ops->tmo_request_launch_time(meta->request.launch_time, - priv); - - if (ops->tmo_request_timestamp) - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - ops->tmo_request_timestamp(priv); - - if (ops->tmo_request_checksum) - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) - ops->tmo_request_checksum(meta->request.csum_start, - meta->request.csum_offset, priv); + compl->tx_timestamp = &meta->completion.tx_timestamp; } /** @@ -231,12 +202,6 @@ static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, { } -static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ -} - static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, const struct xsk_tx_metadata_ops *ops, void *priv) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 46797645a0c2..b344789f5df8 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -245,7 +245,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * details. * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ static inline struct xdp_desc_ctx xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) @@ -260,24 +260,70 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) 0) static inline bool -xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta) +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { - return !(meta->flags & ~XDP_TXMD_FLAGS_VALID); + *flags = READ_ONCE(meta->flags); + if (*flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + if (pool->tx_metadata_len < + offsetofend(struct xsk_tx_metadata, request.launch_time)) + return false; + return !(*flags & ~XDP_TXMD_FLAGS_VALID); +} + +/** + * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission + * and call appropriate xsk_tx_metadata_ops operation. + * @pool: pointer to AF_XDP buffer pool, used to validate the metadata + * @pmeta: pointer to pointer to AF_XDP metadata area + * @ops: pointer to struct xsk_tx_metadata_ops + * @priv: pointer to driver-private area + * + * This function should be called by the networking device when + * it prepares AF_XDP egress packet. + */ +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) +{ + const struct xsk_tx_metadata *meta = *pmeta; + u64 flags; + + if (!meta) + return; + + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) { + *pmeta = NULL; + return; /* no way to signal the error to the user */ + } + + if (ops->tmo_request_launch_time) + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + ops->tmo_request_launch_time( + READ_ONCE(meta->request.launch_time), priv); + + if (ops->tmo_request_timestamp) + if (flags & XDP_TXMD_FLAGS_TIMESTAMP) + ops->tmo_request_timestamp(priv); + + if (ops->tmo_request_checksum) + if (flags & XDP_TXMD_FLAGS_CHECKSUM) + ops->tmo_request_checksum( + READ_ONCE(meta->request.csum_start), + READ_ONCE(meta->request.csum_offset), priv); + + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + *pmeta = NULL; } static inline struct xsk_tx_metadata * __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) { - struct xsk_tx_metadata *meta; - if (!pool->tx_metadata_len) return NULL; - meta = data - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) - return NULL; /* no way to signal the error to the user */ - - return meta; + return data - pool->tx_metadata_len; } static inline struct xsk_tx_metadata * @@ -469,11 +515,20 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) return (struct xdp_desc_ctx){ }; } -static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) +static inline bool +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { return false; } +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) +{ +} + static inline struct xsk_tx_metadata * __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) { diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 58da2f4f4397..cd8643360eb3 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -210,6 +210,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) if (mr->flags & XDP_UMEM_TX_METADATA_LEN) { if (mr->tx_metadata_len >= 256 || mr->tx_metadata_len % 8) return -EINVAL; + if (mr->tx_metadata_len < 16) + return -EINVAL; umem->tx_metadata_len = mr->tx_metadata_len; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index f906d51b6699..7855ee09c4b6 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -967,15 +967,16 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, { struct xsk_tx_metadata *meta = NULL; u16 csum_start, csum_offset; + u64 flags; if (unlikely(pool->tx_metadata_len == 0)) return -EINVAL; meta = buffer - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) return -EINVAL; - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { + if (flags & XDP_TXMD_FLAGS_CHECKSUM) { csum_start = READ_ONCE(meta->request.csum_start); csum_offset = READ_ONCE(meta->request.csum_offset); @@ -996,8 +997,10 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, } } - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - skb->skb_mstamp_ns = meta->request.launch_time; + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + skb->skb_mstamp_ns = READ_ONCE(meta->request.launch_time); + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + meta = NULL; xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); return 0; diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index a4089480b22b..78c14f106395 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -765,11 +765,11 @@ EXPORT_SYMBOL(xp_raw_get_dma); * @addr: desc address (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. - * Saves one call on hotpath, double calculation of the actual address, - * and inline checks for metadata presence and sanity. + * Saves one call on hotpath and double calculation of the actual address. + * Metadata is validated later by xsk_tx_metadata_request(). * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) {