diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 90fea94ad202..53941c6a6875 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -322,6 +322,8 @@ enum { IO_RING_F_DRAIN_DISABLED = BIT(10), IO_RING_F_COMPAT = BIT(11), IO_RING_F_IOWQ_LIMITS_SET = BIT(12), + /* restrictions were inherited from the task that created the ring */ + IO_RING_F_RESTRICT_INHERITED = BIT(13), }; struct iou_ctx {}; diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 911a47075064..9a92a6aa2bcc 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -692,12 +692,11 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) prepare_to_wait(&tctx->wait, &wait, state); io_run_task_work(); io_uring_drop_tctx_refs(current); + /* only skip the sleep for local work this task may run */ xa_for_each(&tctx->xa, index, node) { - if (io_local_work_pending(node->ctx)) { - WARN_ON_ONCE(node->ctx->submitter_task && - node->ctx->submitter_task != current); + if (io_local_work_pending(node->ctx) && + io_allowed_defer_tw_run(node->ctx)) goto end_wait; - } } /* * If we've seen completions, retry without waiting. This diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 7cd411fc4f33..90d4ec7cc761 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -69,8 +69,8 @@ static inline int io_uring_cmd_setsockopt(struct socket *sock, optlen); } -static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, - struct sk_buff *skb, unsigned issue_flags) +static int io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, + struct sk_buff *skb, unsigned int issue_flags) { struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); struct io_uring_cqe cqe[2]; @@ -83,7 +83,7 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, ret = skb_get_tx_timestamp(skb, sk, &ts); if (ret < 0) - return false; + return ret; tskey = serr->ee.ee_data; tstype = serr->ee.ee_info; @@ -98,7 +98,9 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, iots = (struct io_timespec *)&cqe[1]; iots->tv_sec = ts.tv_sec; iots->tv_nsec = ts.tv_nsec; - return io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe); + if (!io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe)) + return -ENOBUFS; + return 0; } static int io_uring_cmd_timestamp(struct socket *sock, @@ -135,7 +137,8 @@ static int io_uring_cmd_timestamp(struct socket *sock, skb = skb_peek(&list); if (!skb) break; - if (!io_process_timestamp_skb(cmd, sk, skb, issue_flags)) + ret = io_process_timestamp_skb(cmd, sk, skb, issue_flags); + if (ret) break; __skb_dequeue(&list); consume_skb(skb); @@ -145,6 +148,12 @@ static int io_uring_cmd_timestamp(struct socket *sock, scoped_guard(spinlock_irqsave, &q->lock) skb_queue_splice(&list, q); } + /* + * Aux CQEs cannot overflow and the poll is edge triggered, so nothing + * re-runs the command once the CQ drains. End the multishot instead. + */ + if (ret == -ENOBUFS) + return -ENOBUFS; return -EAGAIN; } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 3d50608a29c1..0c91d466d825 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2118,8 +2118,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) static void io_rings_free(struct io_ring_ctx *ctx) { - io_free_region(ctx->user, &ctx->sq_region); - io_free_region(ctx->user, &ctx->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &ctx->sq_region); + io_free_region(NULL, &ctx->ring_region); ctx->rings = NULL; RCU_INIT_POINTER(ctx->rings_rcu, NULL); ctx->sq_sqes = NULL; @@ -2525,8 +2526,8 @@ static __cold void io_ring_exit_work(struct work_struct *work) spin_lock(&ctx->completion_lock); spin_unlock(&ctx->completion_lock); - /* pairs with RCU read section in io_req_local_work_add() */ - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + /* pairs with the RCU read sections in the task_work add paths */ + if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL)) synchronize_rcu(); io_ring_ctx_free(ctx); @@ -2870,7 +2871,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); @@ -2884,7 +2886,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES); if (ret) { io_rings_free(ctx); return ret; @@ -3095,7 +3097,8 @@ static void io_ctx_restriction_clone(struct io_ring_ctx *ctx, if (dst->op_registered) ctx->int_flags |= IO_RING_F_OP_RESTRICTED; if (dst->reg_registered) - ctx->int_flags |= IO_RING_F_REG_RESTRICTED; + ctx->int_flags |= IO_RING_F_REG_RESTRICTED | + IO_RING_F_RESTRICT_INHERITED; } static __cold int io_uring_create(struct io_ctx_config *config) diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 7c309173dd19..6e2facf34bfa 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -459,7 +459,7 @@ static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) { if (bl->flags & IOBL_BUF_RING) - io_free_region(ctx->user, &bl->region); + io_free_region(NULL, &bl->region); else io_remove_buffers_legacy(ctx, bl, -1U); @@ -676,7 +676,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = reg.ring_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &bl->region, &rd, mmap_offset); + ret = io_create_region(NULL, &bl->region, &rd, mmap_offset); if (ret) goto fail; br = io_region_get_ptr(&bl->region); @@ -709,7 +709,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) if (!ret) return 0; fail: - io_free_region(ctx->user, &bl->region); + io_free_region(NULL, &bl->region); kfree(bl); return ret; } diff --git a/io_uring/loop.c b/io_uring/loop.c index bbbb6ef14e6a..3d472b7827fa 100644 --- a/io_uring/loop.c +++ b/io_uring/loop.c @@ -2,6 +2,7 @@ #include "io_uring.h" #include "wait.h" #include "loop.h" +#include "tctx.h" static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, const struct iou_loop_params *lp) @@ -81,6 +82,10 @@ int io_run_loop(struct io_ring_ctx *ctx) { int ret; + ret = io_uring_add_tctx_node(ctx); + if (unlikely(ret)) + return ret; + if (!io_allowed_run_tw(ctx)) return -EEXIST; diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 601bb769b235..8d88fc7221af 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -212,7 +212,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr, return 0; } -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset) { @@ -238,8 +238,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, return -EOVERFLOW; nr_pages = reg->size >> PAGE_SHIFT; - if (ctx->user) { - ret = __io_account_mem(ctx->user, nr_pages); + if (user) { + ret = __io_account_mem(user, nr_pages); if (ret) return ret; } @@ -248,7 +248,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, if (reg->flags & IORING_MEM_REGION_TYPE_USER) ret = io_region_pin_pages(mr, reg); else - ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user); + ret = io_region_allocate_pages(mr, reg, mmap_offset, user); if (ret) goto out_free; @@ -257,7 +257,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, goto out_free; return 0; out_free: - io_free_region(ctx->user, mr); + io_free_region(user, mr); return ret; } diff --git a/io_uring/memmap.h b/io_uring/memmap.h index f4cfbb6b9a1f..0714bb5a9616 100644 --- a/io_uring/memmap.h +++ b/io_uring/memmap.h @@ -18,7 +18,7 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, int io_uring_mmap(struct file *file, struct vm_area_struct *vma); void io_free_region(struct user_struct *user, struct io_mapped_region *mr); -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset); diff --git a/io_uring/net.c b/io_uring/net.c index 050ed274170a..f5f4f3ce6ef3 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -1741,7 +1742,8 @@ void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req case AF_INET6: { struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; - if (conn->addr_len < (int)sizeof(*sin6)) + /* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */ + if (conn->addr_len < SIN6_LEN_RFC2133) break; bctx->connect.port = sin6->sin6_port; memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, diff --git a/io_uring/register.c b/io_uring/register.c index 02bc103bcc9d..dab3eb4258df 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -480,8 +480,9 @@ struct io_ring_ctx_rings { static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r) { - io_free_region(ctx->user, &r->sq_region); - io_free_region(ctx->user, &r->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &r->sq_region); + io_free_region(NULL, &r->ring_region); } #define swap_old(ctx, o, n, field) \ @@ -529,7 +530,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING); + ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; @@ -559,7 +560,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES); if (ret) { io_register_free_rings(ctx, &n); return ret; @@ -730,7 +731,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) !(ctx->flags & IORING_SETUP_R_DISABLED)) return -EINVAL; - ret = io_create_region(ctx, ®ion, &rd, IORING_MAP_OFF_PARAM_REGION); + ret = io_create_region(ctx->user, ®ion, &rd, + IORING_MAP_OFF_PARAM_REGION); if (ret) return ret; if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { @@ -764,7 +766,14 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (ctx->submitter_task && ctx->submitter_task != current) return -EEXIST; - if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && !(ctx->flags & IORING_SETUP_R_DISABLED)) { + /* + * A disabled ring is exempt from its own register restrictions, so + * that the creator can set them up and then enable the ring. That + * does not apply to restrictions inherited from the task. + */ + if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && + (!(ctx->flags & IORING_SETUP_R_DISABLED) || + (ctx->int_flags & IO_RING_F_RESTRICT_INHERITED))) { opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); if (!test_bit(opcode, ctx->restrictions.register_op)) return -EACCES; diff --git a/io_uring/tw.c b/io_uring/tw.c index f573bcc3af6a..f9affd68dd26 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -210,6 +210,9 @@ void io_req_normal_work_add(struct io_kiocb *req) struct io_uring_task *tctx = req->tctx; struct io_ring_ctx *ctx = req->ctx; + /* SQPOLL can retire the request on push, see io_ring_exit_work() */ + guard(rcu)(); + /* tw run already pending, nothing else to do */ if (!mpscq_push(&tctx->task_list, &req->io_task_work.node)) return; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 86d580d4410d..cc9df7e64a40 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -374,6 +374,7 @@ struct io_zcrx_args { struct io_kiocb *req; struct io_zcrx_ifq *ifq; unsigned nr_skbs; + bool stopped_early; }; static const struct memory_provider_ops io_uring_pp_zc_ops; @@ -431,7 +432,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, mmap_offset = IORING_MAP_OFF_ZCRX_REGION; mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; - ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset); + ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset); if (ret < 0) return ret; @@ -1899,6 +1900,9 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, } out: + /* Bytes left in len mean an error stopped the walk early. */ + if (len) + args->stopped_early = true; if (offset == start_off) return ret; desc->count -= (offset - start_off); @@ -1935,8 +1939,9 @@ static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq, ret = -ENOTCONN; else ret = -EAGAIN; - } else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) && - (issue_flags & IO_URING_F_MULTISHOT)) { + } else if ((issue_flags & IO_URING_F_MULTISHOT) && + (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) || + args.stopped_early)) { ret = IOU_REQUEUE; } else if (sock_flag(sk, SOCK_DONE)) { /* Make it to retry until it finally gets 0. */