github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 04:34:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'xsk-harden-tx-metadata-validation-against-races'

Browse Source 
Stanislav Fomichev says:

====================
xsk: harden TX metadata validation against races

Cen Zhang reported a KASAN out-of-bounds read when AF_XDP is configured
with a TX metadata area smaller than struct xsk_tx_metadata. The metadata
is also shared with user space, so reading its flags more than once can
produce inconsistent validation and processing decisions.

Require enough space for the flags and one request field, validate the
launch-time field against the configured metadata length, and use one
snapshot of the flags while processing each request. Carry the validated
decision through completion handling so later user-space changes cannot
enable an unrequested completion timestamp.

Reported-by: AutonomousCodeSecurity@microsoft.com
Reported-by: Cen Zhang (Microsoft) <blbllhy@gmail.com>
Link: https://lore.kernel.org/netdev/20260720155210.34229-1-blbllhy@gmail.com/
====================

Link: https://patch.msgid.link/20260727161959.885642-1-sdf@fomichev.me
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2026-08-03 16:43:56 -07:00
commit 5243c3e3ad
12 changed files with 102 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/net/ethernet/intel/igc/igc_main.c
View File

@ -3082,7 +3082,7 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring)
meta_req.tx_buffer = bi;
meta_req.meta = meta;
meta_req.used_desc = 0;
xsk_tx_metadata_request(meta, &igc_xsk_tx_metadata_ops,
xsk_tx_metadata_request(pool, &meta, &igc_xsk_tx_metadata_ops,
&meta_req);
/* xsk_tx_metadata_request() may have updated next_to_use */

2
drivers/net/ethernet/mellanox/mlx5/core/en.h
View File

@ -483,7 +483,7 @@ typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
struct mlx5e_xmit_data *,
int,
struct xsk_tx_metadata *);
struct xsk_tx_metadata **);
struct mlx5e_xdpsq {
/* data path */

15
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
View File

@ -452,11 +452,11 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq
INDIRECT_CALLABLE_SCOPE bool
mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
int check_result, struct xsk_tx_metadata *meta);
int check_result, struct xsk_tx_metadata **meta);
INDIRECT_CALLABLE_SCOPE bool
mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
int check_result, struct xsk_tx_metadata *meta)
int check_result, struct xsk_tx_metadata **meta)
{
struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
struct mlx5e_xdpsq_stats *stats = sq->stats;
@ -504,7 +504,10 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx
* and it's safe to complete it at any time.
*/
mlx5e_xdp_mpwqe_session_start(sq);
xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, &session->wqe->eth);
if (meta)
xsk_tx_metadata_request(sq->xsk_pool, meta,
&mlx5e_xsk_tx_metadata_ops,
&session->wqe->eth);
}
mlx5e_xdp_mpwqe_add_dseg(sq, p, stats);
@ -535,7 +538,7 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
INDIRECT_CALLABLE_SCOPE bool
mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
int check_result, struct xsk_tx_metadata *meta)
int check_result, struct xsk_tx_metadata **meta)
{
struct mlx5e_xmit_data_frags *xdptxdf =
container_of(xdptxd, struct mlx5e_xmit_data_frags, xd);
@ -649,7 +652,9 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
sq->pc += num_wqebbs;
xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, eseg);
if (meta)
xsk_tx_metadata_request(sq->xsk_pool, meta,
&mlx5e_xsk_tx_metadata_ops, eseg);
sq->doorbell_cseg = cseg;

4
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
View File

@ -114,11 +114,11 @@ extern const struct xsk_tx_metadata_ops mlx5e_xsk_tx_metadata_ops;
INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
struct mlx5e_xmit_data *xdptxd,
int check_result,
struct xsk_tx_metadata *meta));
struct xsk_tx_metadata **meta));
INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
struct mlx5e_xmit_data *xdptxd,
int check_result,
struct xsk_tx_metadata *meta));
struct xsk_tx_metadata **meta));
INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));
INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));

2
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
View File

@ -105,7 +105,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
mlx5e_xmit_xdp_frame, sq, &xdptxd,
check_result, meta);
check_result, &meta);
if (unlikely(!ret)) {
if (sq->mpwqe.wqe)
mlx5e_xdp_mpwqe_complete(sq);

4
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
View File

@ -2748,8 +2748,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
meta_req.set_ic = &set_ic;
meta_req.tbs = tx_q->tbs;
meta_req.edesc = &tx_q->dma_entx[entry];
xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops,
&meta_req);
xsk_tx_metadata_request(pool, &meta,
&stmmac_xsk_tx_metadata_ops, &meta_req);
if (set_ic) {
tx_q->tx_count_frames = 0;
stmmac_set_tx_ic(priv, tx_desc);

2
include/net/libeth/xsk.h
View File

@ -205,7 +205,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc,
BUILD_BUG_ON(!__builtin_constant_p(tmo == libeth_xsktmo));
tmo = tmo == libeth_xsktmo ? &__libeth_xsktmo : tmo;
xsk_tx_metadata_request(ctx.meta, tmo, &desc);
xsk_tx_metadata_request(sq->pool, &ctx.meta, tmo, &desc);
return desc;
}

47
include/net/xdp_sock.h
View File

@ -141,45 +141,16 @@ INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));
static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
struct xsk_tx_metadata_compl *compl)
{
compl->tx_timestamp = NULL;
if (!meta)
return;
if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
compl->tx_timestamp = &meta->completion.tx_timestamp;
else
compl->tx_timestamp = NULL;
}
/* we can only arrive here if the completion timestamp has been
* requested via XDP_TXMD_FLAGS_TIMESTAMP, see xsk_tx_metadata_request
*/
/**
* xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
* and call appropriate xsk_tx_metadata_ops operation.
* @meta: pointer to AF_XDP metadata area
* @ops: pointer to struct xsk_tx_metadata_ops
* @priv: pointer to driver-private aread
*
* This function should be called by the networking device when
* it prepares AF_XDP egress packet.
*/
static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
const struct xsk_tx_metadata_ops *ops,
void *priv)
{
if (!meta)
return;
if (ops->tmo_request_launch_time)
if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
ops->tmo_request_launch_time(meta->request.launch_time,
priv);
if (ops->tmo_request_timestamp)
if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
ops->tmo_request_timestamp(priv);
if (ops->tmo_request_checksum)
if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
ops->tmo_request_checksum(meta->request.csum_start,
meta->request.csum_offset, priv);
compl->tx_timestamp = &meta->completion.tx_timestamp;
}
/**
@ -231,12 +202,6 @@ static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
{
}
static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
const struct xsk_tx_metadata_ops *ops,
void *priv)
{
}
static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
const struct xsk_tx_metadata_ops *ops,
void *priv)

77
include/net/xdp_sock_drv.h
View File

@ -245,7 +245,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
* details.
*
* Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata
* pointer, if it is present and valid (initialized to %NULL otherwise).
* pointer, if it is present (initialized to %NULL otherwise).
*/
static inline struct xdp_desc_ctx
xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
@ -260,24 +260,70 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
0)
static inline bool
xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta)
xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool,
const struct xsk_tx_metadata *meta, u64 *flags)
{
return !(meta->flags & ~XDP_TXMD_FLAGS_VALID);
*flags = READ_ONCE(meta->flags);
if (*flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
if (pool->tx_metadata_len <
offsetofend(struct xsk_tx_metadata, request.launch_time))
return false;
return !(*flags & ~XDP_TXMD_FLAGS_VALID);
}
/**
* xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
* and call appropriate xsk_tx_metadata_ops operation.
* @pool: pointer to AF_XDP buffer pool, used to validate the metadata
* @pmeta: pointer to pointer to AF_XDP metadata area
* @ops: pointer to struct xsk_tx_metadata_ops
* @priv: pointer to driver-private area
*
* This function should be called by the networking device when
* it prepares AF_XDP egress packet.
*/
static inline void
xsk_tx_metadata_request(const struct xsk_buff_pool *pool,
struct xsk_tx_metadata **pmeta,
const struct xsk_tx_metadata_ops *ops, void *priv)
{
const struct xsk_tx_metadata *meta = *pmeta;
u64 flags;
if (!meta)
return;
if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) {
*pmeta = NULL;
return; /* no way to signal the error to the user */
}
if (ops->tmo_request_launch_time)
if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
ops->tmo_request_launch_time(
READ_ONCE(meta->request.launch_time), priv);
if (ops->tmo_request_timestamp)
if (flags & XDP_TXMD_FLAGS_TIMESTAMP)
ops->tmo_request_timestamp(priv);
if (ops->tmo_request_checksum)
if (flags & XDP_TXMD_FLAGS_CHECKSUM)
ops->tmo_request_checksum(
READ_ONCE(meta->request.csum_start),
READ_ONCE(meta->request.csum_offset), priv);
if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP))
*pmeta = NULL;
}
static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
{
struct xsk_tx_metadata *meta;
if (!pool->tx_metadata_len)
return NULL;
meta = data - pool->tx_metadata_len;
if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
return NULL; /* no way to signal the error to the user */
return meta;
return data - pool->tx_metadata_len;
}
static inline struct xsk_tx_metadata *
@ -469,11 +515,20 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
return (struct xdp_desc_ctx){ };
}
static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta)
static inline bool
xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool,
const struct xsk_tx_metadata *meta, u64 *flags)
{
return false;
}
static inline void
xsk_tx_metadata_request(const struct xsk_buff_pool *pool,
struct xsk_tx_metadata **pmeta,
const struct xsk_tx_metadata_ops *ops, void *priv)
{
}
static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
{

2
net/xdp/xdp_umem.c
View File

@ -210,6 +210,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
if (mr->flags & XDP_UMEM_TX_METADATA_LEN) {
if (mr->tx_metadata_len >= 256 || mr->tx_metadata_len % 8)
return -EINVAL;
if (mr->tx_metadata_len < 16)
return -EINVAL;
umem->tx_metadata_len = mr->tx_metadata_len;
}

11
net/xdp/xsk.c
View File

@ -967,15 +967,16 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
{
struct xsk_tx_metadata *meta = NULL;
u16 csum_start, csum_offset;
u64 flags;
if (unlikely(pool->tx_metadata_len == 0))
return -EINVAL;
meta = buffer - pool->tx_metadata_len;
if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags)))
return -EINVAL;
if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
if (flags & XDP_TXMD_FLAGS_CHECKSUM) {
csum_start = READ_ONCE(meta->request.csum_start);
csum_offset = READ_ONCE(meta->request.csum_offset);
@ -996,8 +997,10 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
}
}
if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
skb->skb_mstamp_ns = meta->request.launch_time;
if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
skb->skb_mstamp_ns = READ_ONCE(meta->request.launch_time);
if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP))
meta = NULL;
xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
return 0;

6
net/xdp/xsk_buff_pool.c
View File

@ -765,11 +765,11 @@ EXPORT_SYMBOL(xp_raw_get_dma);
* @addr: desc address (from userspace)
*
* Helper for getting desc's DMA address and metadata pointer, if present.
* Saves one call on hotpath, double calculation of the actual address,
* and inline checks for metadata presence and sanity.
* Saves one call on hotpath and double calculation of the actual address.
* Metadata is validated later by xsk_tx_metadata_request().
*
* Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata
* pointer, if it is present and valid (initialized to %NULL otherwise).
* pointer, if it is present (initialized to %NULL otherwise).
*/
struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
{