diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 7ac9eb6a0934..1e194de7d6d2 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -44,6 +44,17 @@ static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *nio return container_of(owner, struct io_zcrx_area, nia); } +static bool zcrx_set_ring_ctx(struct io_zcrx_ifq *zcrx, + struct io_ring_ctx *ctx) +{ + guard(spinlock_bh)(&zcrx->ctx_lock); + if (zcrx->master_ctx) + return false; + percpu_ref_get(&ctx->refs); + zcrx->master_ctx = ctx; + return true; +} + static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) { struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); @@ -529,6 +540,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) return NULL; ifq->if_rxq = -1; + spin_lock_init(&ifq->ctx_lock); spin_lock_init(&ifq->rq.lock); mutex_init(&ifq->pp_lock); refcount_set(&ifq->refs, 1); @@ -578,6 +590,8 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) return; if (WARN_ON_ONCE(ifq->netdev != NULL)) return; + if (WARN_ON_ONCE(ifq->master_ctx)) + return; if (ifq->area) io_zcrx_free_area(ifq, ifq->area); @@ -654,17 +668,24 @@ static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) } } -static void zcrx_unregister_user(struct io_zcrx_ifq *ifq) +static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) { + scoped_guard(spinlock_bh, &ifq->ctx_lock) { + if (ctx && ifq->master_ctx == ctx) { + ifq->master_ctx = NULL; + percpu_ref_put(&ctx->refs); + } + } + if (refcount_dec_and_test(&ifq->user_refs)) { io_close_queue(ifq); io_zcrx_scrub(ifq); } } -static void zcrx_unregister(struct io_zcrx_ifq *ifq) +static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) { - zcrx_unregister_user(ifq); + zcrx_unregister_user(ifq, ctx); io_put_zcrx_ifq(ifq); } @@ -684,7 +705,7 @@ static int zcrx_box_release(struct inode *inode, struct file *file) if (WARN_ON_ONCE(!ifq)) return -EFAULT; - zcrx_unregister(ifq); + zcrx_unregister(ifq, NULL); return 0; } @@ -709,7 +730,7 @@ static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops, ifq, O_CLOEXEC, NULL); if (IS_ERR(file)) { - zcrx_unregister(ifq); + zcrx_unregister(ifq, NULL); return PTR_ERR(file); } @@ -785,7 +806,7 @@ static int import_zcrx(struct io_ring_ctx *ctx, scoped_guard(mutex, &ctx->mmap_lock) xa_erase(&ctx->zcrx_ctxs, id); err: - zcrx_unregister(ifq); + zcrx_unregister(ifq, ctx); return ret; } @@ -930,12 +951,14 @@ int io_register_zcrx(struct io_ring_ctx *ctx, ret = -EFAULT; goto err; } + + zcrx_set_ring_ctx(ifq, ctx); return 0; err: scoped_guard(mutex, &ctx->mmap_lock) xa_erase(&ctx->zcrx_ctxs, id); ifq_free: - zcrx_unregister(ifq); + zcrx_unregister(ifq, ctx); return ret; } @@ -965,7 +988,7 @@ void io_terminate_zcrx(struct io_ring_ctx *ctx) break; set_zcrx_entry_mark(ctx, id); id++; - zcrx_unregister_user(ifq); + zcrx_unregister_user(ifq, ctx); } } diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 75e0a4e6ef6e..76389a5dd50f 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -72,6 +72,9 @@ struct io_zcrx_ifq { */ struct mutex pp_lock; struct io_mapped_region rq_region; + + spinlock_t ctx_lock; + struct io_ring_ctx *master_ctx; }; #if defined(CONFIG_IO_URING_ZCRX)