github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'io_uring-7.3' into for-7.4/io_uring

Browse Source 
Merge 7.3 fixes to avoid conflicts with future changes merged for the
7.4 kernel.

* io_uring-7.3:
  io_uring/cancel: don't spin on local work the task can't run
  io_uring/net: populate CONNECT filter fields for 24-byte sockaddr_in6
  io_uring/register: don't exempt disabled rings from task restrictions
  io_uring/kbuf: don't charge provided buffer rings to RLIMIT_MEMLOCK
  io_uring: do not charge the SQ/CQ rings to RLIMIT_MEMLOCK
  io_uring: fix task_work add use-after-free with SQPOLL
  io_uring/cmd_net: end TX_TIMESTAMP multishot when the CQ is full
  io_uring/zcrx: requeue multishot receives stopped by a local resource
  io_uring: initialize task context before running the BPF loop
This commit is contained in:
Jens Axboe 2026-10-15 13:44:51 -06:00
commit 0870eb27a8
12 changed files with 72 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/io_uring_types.h
View File

@ -322,6 +322,8 @@ enum {
IO_RING_F_DRAIN_DISABLED = BIT(10),
IO_RING_F_COMPAT = BIT(11),
IO_RING_F_IOWQ_LIMITS_SET = BIT(12),
/* restrictions were inherited from the task that created the ring */
IO_RING_F_RESTRICT_INHERITED = BIT(13),
};
struct iou_ctx {};

7
io_uring/cancel.c
View File

@ -692,12 +692,11 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
prepare_to_wait(&tctx->wait, &wait, state);
io_run_task_work();
io_uring_drop_tctx_refs(current);
/* only skip the sleep for local work this task may run */
xa_for_each(&tctx->xa, index, node) {
if (io_local_work_pending(node->ctx)) {
WARN_ON_ONCE(node->ctx->submitter_task &&
node->ctx->submitter_task != current);
if (io_local_work_pending(node->ctx) &&
io_allowed_defer_tw_run(node->ctx))
goto end_wait;
}
}
/*
* If we've seen completions, retry without waiting. This

19
io_uring/cmd_net.c
View File

@ -69,8 +69,8 @@ static inline int io_uring_cmd_setsockopt(struct socket *sock,
optlen);
}
static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
struct sk_buff *skb, unsigned issue_flags)
static int io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
struct sk_buff *skb, unsigned int issue_flags)
{
struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
struct io_uring_cqe cqe[2];
@ -83,7 +83,7 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
ret = skb_get_tx_timestamp(skb, sk, &ts);
if (ret < 0)
return false;
return ret;
tskey = serr->ee.ee_data;
tstype = serr->ee.ee_info;
@ -98,7 +98,9 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
iots = (struct io_timespec *)&cqe[1];
iots->tv_sec = ts.tv_sec;
iots->tv_nsec = ts.tv_nsec;
return io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe);
if (!io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe))
return -ENOBUFS;
return 0;
}
static int io_uring_cmd_timestamp(struct socket *sock,
@ -135,7 +137,8 @@ static int io_uring_cmd_timestamp(struct socket *sock,
skb = skb_peek(&list);
if (!skb)
break;
if (!io_process_timestamp_skb(cmd, sk, skb, issue_flags))
ret = io_process_timestamp_skb(cmd, sk, skb, issue_flags);
if (ret)
break;
__skb_dequeue(&list);
consume_skb(skb);
@ -145,6 +148,12 @@ static int io_uring_cmd_timestamp(struct socket *sock,
scoped_guard(spinlock_irqsave, &q->lock)
skb_queue_splice(&list, q);
}
/*
* Aux CQEs cannot overflow and the poll is edge triggered, so nothing
* re-runs the command once the CQ drains. End the multishot instead.
*/
if (ret == -ENOBUFS)
return -ENOBUFS;
return -EAGAIN;
}

17
io_uring/io_uring.c
View File

@ -2118,8 +2118,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
static void io_rings_free(struct io_ring_ctx *ctx)
{
io_free_region(ctx->user, &ctx->sq_region);
io_free_region(ctx->user, &ctx->ring_region);
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
io_free_region(NULL, &ctx->sq_region);
io_free_region(NULL, &ctx->ring_region);
ctx->rings = NULL;
RCU_INIT_POINTER(ctx->rings_rcu, NULL);
ctx->sq_sqes = NULL;
@ -2525,8 +2526,8 @@ static __cold void io_ring_exit_work(struct work_struct *work)
spin_lock(&ctx->completion_lock);
spin_unlock(&ctx->completion_lock);
/* pairs with RCU read section in io_req_local_work_add() */
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
/* pairs with the RCU read sections in the task_work add paths */
if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL))
synchronize_rcu();
io_ring_ctx_free(ctx);
@ -2870,7 +2871,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
if (ret)
return ret;
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
@ -2884,7 +2886,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region(ctx, &ctx->sq_region, &rd, IORING_OFF_SQES);
ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES);
if (ret) {
io_rings_free(ctx);
return ret;
@ -3095,7 +3097,8 @@ static void io_ctx_restriction_clone(struct io_ring_ctx *ctx,
if (dst->op_registered)
ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
if (dst->reg_registered)
ctx->int_flags |= IO_RING_F_REG_RESTRICTED;
ctx->int_flags |= IO_RING_F_REG_RESTRICTED |
IO_RING_F_RESTRICT_INHERITED;
}
static __cold int io_uring_create(struct io_ctx_config *config)

6
io_uring/kbuf.c
View File

@ -459,7 +459,7 @@ static int io_remove_buffers_legacy(struct io_ring_ctx *ctx,
static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
{
if (bl->flags & IOBL_BUF_RING)
io_free_region(ctx->user, &bl->region);
io_free_region(NULL, &bl->region);
else
io_remove_buffers_legacy(ctx, bl, -1U);
@ -676,7 +676,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
rd.user_addr = reg.ring_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region(ctx, &bl->region, &rd, mmap_offset);
ret = io_create_region(NULL, &bl->region, &rd, mmap_offset);
if (ret)
goto fail;
br = io_region_get_ptr(&bl->region);
@ -709,7 +709,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
if (!ret)
return 0;
fail:
io_free_region(ctx->user, &bl->region);
io_free_region(NULL, &bl->region);
kfree(bl);
return ret;
}

5
io_uring/loop.c
View File

@ -2,6 +2,7 @@
#include "io_uring.h"
#include "wait.h"
#include "loop.h"
#include "tctx.h"
static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
const struct iou_loop_params *lp)
@ -81,6 +82,10 @@ int io_run_loop(struct io_ring_ctx *ctx)
{
int ret;
ret = io_uring_add_tctx_node(ctx);
if (unlikely(ret))
return ret;
if (!io_allowed_run_tw(ctx))
return -EEXIST;

10
io_uring/memmap.c
View File

@ -212,7 +212,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr,
return 0;
}
int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
int io_create_region(struct user_struct *user, struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset)
{
@ -238,8 +238,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
return -EOVERFLOW;
nr_pages = reg->size >> PAGE_SHIFT;
if (ctx->user) {
ret = __io_account_mem(ctx->user, nr_pages);
if (user) {
ret = __io_account_mem(user, nr_pages);
if (ret)
return ret;
}
@ -248,7 +248,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
if (reg->flags & IORING_MEM_REGION_TYPE_USER)
ret = io_region_pin_pages(mr, reg);
else
ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user);
ret = io_region_allocate_pages(mr, reg, mmap_offset, user);
if (ret)
goto out_free;
@ -257,7 +257,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
goto out_free;
return 0;
out_free:
io_free_region(ctx->user, mr);
io_free_region(user, mr);
return ret;
}

2
io_uring/memmap.h
View File

@ -18,7 +18,7 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
int io_uring_mmap(struct file *file, struct vm_area_struct *vma);
void io_free_region(struct user_struct *user, struct io_mapped_region *mr);
int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
int io_create_region(struct user_struct *user, struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset);

4
io_uring/net.c
View File

@ -7,6 +7,7 @@
#include <linux/un.h>
#include <linux/compat.h>
#include <net/compat.h>
#include <net/ipv6.h>
#include <linux/io_uring.h>
#include <uapi/linux/io_uring.h>
@ -1741,7 +1742,8 @@ void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req
case AF_INET6: {
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
if (conn->addr_len < (int)sizeof(*sin6))
/* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */
if (conn->addr_len < SIN6_LEN_RFC2133)
break;
bctx->connect.port = sin6->sin6_port;
memcpy(bctx->connect.v6_addr, &sin6->sin6_addr,

21
io_uring/register.c
View File

@ -480,8 +480,9 @@ struct io_ring_ctx_rings {
static void io_register_free_rings(struct io_ring_ctx *ctx,
struct io_ring_ctx_rings *r)
{
io_free_region(ctx->user, &r->sq_region);
io_free_region(ctx->user, &r->ring_region);
/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
io_free_region(NULL, &r->sq_region);
io_free_region(NULL, &r->ring_region);
}
#define swap_old(ctx, o, n, field) \
@ -529,7 +530,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING);
if (ret)
return ret;
@ -559,7 +560,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES);
if (ret) {
io_register_free_rings(ctx, &n);
return ret;
@ -730,7 +731,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
!(ctx->flags & IORING_SETUP_R_DISABLED))
return -EINVAL;
ret = io_create_region(ctx, &region, &rd, IORING_MAP_OFF_PARAM_REGION);
ret = io_create_region(ctx->user, &region, &rd,
IORING_MAP_OFF_PARAM_REGION);
if (ret)
return ret;
if (copy_to_user(rd_uptr, &rd, sizeof(rd))) {
@ -764,7 +766,14 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
if (ctx->submitter_task && ctx->submitter_task != current)
return -EEXIST;
if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && !(ctx->flags & IORING_SETUP_R_DISABLED)) {
/*
* A disabled ring is exempt from its own register restrictions, so
* that the creator can set them up and then enable the ring. That
* does not apply to restrictions inherited from the task.
*/
if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) &&
(!(ctx->flags & IORING_SETUP_R_DISABLED) ||
(ctx->int_flags & IO_RING_F_RESTRICT_INHERITED))) {
opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
if (!test_bit(opcode, ctx->restrictions.register_op))
return -EACCES;

3
io_uring/tw.c
View File

@ -210,6 +210,9 @@ void io_req_normal_work_add(struct io_kiocb *req)
struct io_uring_task *tctx = req->tctx;
struct io_ring_ctx *ctx = req->ctx;
/* SQPOLL can retire the request on push, see io_ring_exit_work() */
guard(rcu)();
/* tw run already pending, nothing else to do */
if (!mpscq_push(&tctx->task_list, &req->io_task_work.node))
return;

11
io_uring/zcrx.c
View File

@ -374,6 +374,7 @@ struct io_zcrx_args {
struct io_kiocb *req;
struct io_zcrx_ifq *ifq;
unsigned nr_skbs;
bool stopped_early;
};
static const struct memory_provider_ops io_uring_pp_zc_ops;
@ -431,7 +432,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
mmap_offset = IORING_MAP_OFF_ZCRX_REGION;
mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT;
ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset);
ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset);
if (ret < 0)
return ret;
@ -1899,6 +1900,9 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
}
out:
/* Bytes left in len mean an error stopped the walk early. */
if (len)
args->stopped_early = true;
if (offset == start_off)
return ret;
desc->count -= (offset - start_off);
@ -1935,8 +1939,9 @@ static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
ret = -ENOTCONN;
else
ret = -EAGAIN;
} else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) &&
(issue_flags & IO_URING_F_MULTISHOT)) {
} else if ((issue_flags & IO_URING_F_MULTISHOT) &&
(unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) ||
args.stopped_early)) {
ret = IOU_REQUEUE;
} else if (sock_flag(sk, SOCK_DONE)) {
/* Make it to retry until it finally gets 0. */