github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-02 03:24:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'for-6.15/io_uring' into for-6.15/io_uring-reg-vec

Browse Source 
* for-6.15/io_uring: (80 commits)
  io_uring: introduce io_cache_free() helper
  io_uring/rsrc: skip NULL file/buffer checks in io_free_rsrc_node()
  io_uring/rsrc: avoid NULL node check on io_sqe_buffer_register() failure
  io_uring/rsrc: call io_free_node() on io_sqe_buffer_register() failure
  io_uring/rsrc: free io_rsrc_node using kfree()
  io_uring/rsrc: split out io_free_node() helper
  io_uring/rsrc: include io_uring_types.h in rsrc.h
  ublk: don't cast registered buffer index to int
  io_uring/nop: use io_find_buf_node()
  io_uring/rsrc: declare io_find_buf_node() in header file
  io_uring/ublk: report error when unregister operation fails
  io_uring: convert cmd_to_io_kiocb() macro to function
  io_uring/uring_cmd: specify io_uring_cmd_import_fixed() pointer type
  io_uring/rsrc: use rq_data_dir() to compute bvec dir
  selftests: ublk: add ublk zero copy test
  selftests: ublk: add file backed ublk
  selftests: ublk: add kernel selftests for ublk
  io_uring: cache nodes and mapped buffers
  ublk: zc register/unregister bvec
  io_uring: add support for kernel registered bvecs
  ...
This commit is contained in:
Jens Axboe 2025-03-07 09:07:07 -07:00
commit 94765d71a0
47 changed files with 2983 additions and 842 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MAINTAINERS
View File

@ -24253,6 +24253,7 @@ S: Maintained
F: Documentation/block/ublk.rst
F: drivers/block/ublk_drv.c
F: include/uapi/linux/ublk_cmd.h
F: tools/testing/selftests/ublk/
UBSAN
M: Kees Cook <kees@kernel.org>

56
drivers/block/ublk_drv.c
View File

@ -51,6 +51,9 @@
/* private ioctl command mirror */
#define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)
#define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)
#define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)
/* All UBLK_F_* have to be included into UBLK_F_ALL */
#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
| UBLK_F_URING_CMD_COMP_IN_TASK \
@ -196,12 +199,14 @@ struct ublk_params_header {
static bool ublk_abort_requests(struct ublk_device *ub, struct ublk_queue *ubq);
static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
struct ublk_queue *ubq, int tag, size_t offset);
static inline unsigned int ublk_req_build_flags(struct request *req);
static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
int tag);
static inline bool ublk_dev_is_user_copy(const struct ublk_device *ub)
{
return ub->dev_info.flags & UBLK_F_USER_COPY;
return ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);
}
static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
@ -581,7 +586,7 @@ static void ublk_apply_params(struct ublk_device *ub)
static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)
{
return ubq->flags & UBLK_F_USER_COPY;
return ubq->flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);
}
static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)
@ -1747,6 +1752,42 @@ static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
io_uring_cmd_mark_cancelable(cmd, issue_flags);
}
static void ublk_io_release(void *priv)
{
struct request *rq = priv;
struct ublk_queue *ubq = rq->mq_hctx->driver_data;
ublk_put_req_ref(ubq, rq);
}
static int ublk_register_io_buf(struct io_uring_cmd *cmd,
struct ublk_queue *ubq, unsigned int tag,
unsigned int index, unsigned int issue_flags)
{
struct ublk_device *ub = cmd->file->private_data;
struct request *req;
int ret;
req = __ublk_check_and_get_req(ub, ubq, tag, 0);
if (!req)
return -EINVAL;
ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,
issue_flags);
if (ret) {
ublk_put_req_ref(ubq, req);
return ret;
}
return 0;
}
static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,
unsigned int index, unsigned int issue_flags)
{
return io_buffer_unregister_bvec(cmd, index, issue_flags);
}
static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
unsigned int issue_flags,
const struct ublksrv_io_cmd *ub_cmd)
@ -1798,6 +1839,10 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
ret = -EINVAL;
switch (_IOC_NR(cmd_op)) {
case UBLK_IO_REGISTER_IO_BUF:
return ublk_register_io_buf(cmd, ubq, tag, ub_cmd->addr, issue_flags);
case UBLK_IO_UNREGISTER_IO_BUF:
return ublk_unregister_io_buf(cmd, ub_cmd->addr, issue_flags);
case UBLK_IO_FETCH_REQ:
/* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
if (ublk_queue_ready(ubq)) {
@ -2459,7 +2504,7 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
* buffer by pwrite() to ublk char device, which can't be
* used for unprivileged device
*/
if (info.flags & UBLK_F_USER_COPY)
if (info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY))
return -EINVAL;
}
@ -2527,9 +2572,6 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
goto out_free_dev_number;
}
/* We are not ready to support zero copy */
ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
ub->dev_info.nr_hw_queues = min_t(unsigned int,
ub->dev_info.nr_hw_queues, nr_cpu_ids);
ublk_align_max_io_size(ub);
@ -2860,7 +2902,7 @@ static int ublk_ctrl_get_features(struct io_uring_cmd *cmd)
{
const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
void __user *argp = (void __user *)(unsigned long)header->addr;
u64 features = UBLK_F_ALL & ~UBLK_F_SUPPORT_ZERO_COPY;
u64 features = UBLK_F_ALL;
if (header->len != UBLK_FEATURES_LEN || !header->addr)
return -EINVAL;

12
drivers/nvme/host/ioctl.c
View File

@ -114,7 +114,8 @@ static struct request *nvme_alloc_user_request(struct request_queue *q,
static int nvme_map_user_request(struct request *req, u64 ubuffer,
unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
struct io_uring_cmd *ioucmd, unsigned int flags)
struct io_uring_cmd *ioucmd, unsigned int flags,
unsigned int iou_issue_flags)
{
struct request_queue *q = req->q;
struct nvme_ns *ns = q->queuedata;
@ -142,7 +143,8 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
if (WARN_ON_ONCE(flags & NVME_IOCTL_VEC))
return -EINVAL;
ret = io_uring_cmd_import_fixed(ubuffer, bufflen,
rq_data_dir(req), &iter, ioucmd);
rq_data_dir(req), &iter, ioucmd,
iou_issue_flags);
if (ret < 0)
goto out;
ret = blk_rq_map_user_iov(q, req, NULL, &iter, GFP_KERNEL);
@ -194,7 +196,7 @@ static int nvme_submit_user_cmd(struct request_queue *q,
req->timeout = timeout;
if (ubuffer && bufflen) {
ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,
meta_len, NULL, flags);
meta_len, NULL, flags, 0);
if (ret)
return ret;
}
@ -510,10 +512,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
return PTR_ERR(req);
req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;
if (d.addr && d.data_len) {
if (d.data_len) {
ret = nvme_map_user_request(req, d.addr,
d.data_len, nvme_to_user_ptr(d.metadata),
d.metadata_len, ioucmd, vec);
d.metadata_len, ioucmd, vec, issue_flags);
if (ret)
return ret;
}

17
include/linux/io_uring/cmd.h
View File

@ -4,6 +4,7 @@
#include <uapi/linux/io_uring.h>
#include <linux/io_uring_types.h>
#include <linux/blk-mq.h>
/* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */
#define IORING_URING_CMD_CANCELABLE (1U << 30)
@ -39,7 +40,9 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz)
#if defined(CONFIG_IO_URING)
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd);
struct iov_iter *iter,
struct io_uring_cmd *ioucmd,
unsigned int issue_flags);
/*
* Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
@ -66,8 +69,10 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd);
#else
static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd)
static inline int
io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, struct io_uring_cmd *ioucmd,
unsigned int issue_flags)
{
return -EOPNOTSUPP;
}
@ -123,4 +128,10 @@ static inline struct io_uring_cmd_data *io_uring_cmd_get_async_data(struct io_ur
return cmd_to_io_kiocb(cmd)->async_data;
}
int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
void (*release)(void *), unsigned int index,
unsigned int issue_flags);
int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
unsigned int issue_flags);
#endif /* _LINUX_IO_URING_CMD_H */

20
include/linux/io_uring_types.h
View File

@ -292,6 +292,8 @@ struct io_ring_ctx {
struct io_file_table file_table;
struct io_rsrc_data buf_table;
struct io_alloc_cache node_cache;
struct io_alloc_cache imu_cache;
struct io_submit_state submit_state;
@ -360,7 +362,6 @@ struct io_ring_ctx {
spinlock_t completion_lock;
struct list_head io_buffers_comp;
struct list_head cq_overflow_list;
struct hlist_head waitid_list;
@ -379,8 +380,6 @@ struct io_ring_ctx {
unsigned int file_alloc_start;
unsigned int file_alloc_end;
struct list_head io_buffers_cache;
/* Keep this last, we don't need it for the fast path */
struct wait_queue_head poll_wq;
struct io_restriction restrictions;
@ -439,8 +438,15 @@ struct io_ring_ctx {
struct io_mapped_region param_region;
};
/*
* Token indicating function is called in task work context:
* ctx->uring_lock is held and any completions generated will be flushed.
* ONLY core io_uring.c should instantiate this struct.
*/
struct io_tw_state {
};
/* Alias to use in code that doesn't instantiate struct io_tw_state */
typedef struct io_tw_state io_tw_token_t;
enum {
REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
@ -566,7 +572,7 @@ enum {
REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
};
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
struct io_task_work {
struct llist_node node;
@ -601,7 +607,11 @@ static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \
((cmd_type *)&(req)->cmd) \
)
#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr)
static inline struct io_kiocb *cmd_to_io_kiocb(void *ptr)
{
return ptr;
}
struct io_kiocb {
union {

4
include/uapi/linux/ublk_cmd.h
View File

@ -94,6 +94,10 @@
_IOWR('u', UBLK_IO_COMMIT_AND_FETCH_REQ, struct ublksrv_io_cmd)
#define UBLK_U_IO_NEED_GET_DATA \
_IOWR('u', UBLK_IO_NEED_GET_DATA, struct ublksrv_io_cmd)
#define UBLK_U_IO_REGISTER_IO_BUF \
_IOWR('u', 0x23, struct ublksrv_io_cmd)
#define UBLK_U_IO_UNREGISTER_IO_BUF \
_IOWR('u', 0x24, struct ublksrv_io_cmd)
/* only ABORT means that no re-fetch */
#define UBLK_IO_RES_OK 0

6
io_uring/alloc_cache.h
View File

@ -68,4 +68,10 @@ static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)
return io_cache_alloc_new(cache, gfp);
}
static inline void io_cache_free(struct io_alloc_cache *cache, void *obj)
{
if (!io_alloc_cache_put(cache, obj))
kfree(obj);
}
#endif

42
io_uring/cancel.c
View File

@ -341,3 +341,45 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
fput(file);
return ret;
}
bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
struct hlist_head *list, bool cancel_all,
bool (*cancel)(struct io_kiocb *))
{
struct hlist_node *tmp;
struct io_kiocb *req;
bool found = false;
lockdep_assert_held(&ctx->uring_lock);
hlist_for_each_entry_safe(req, tmp, list, hash_node) {
if (!io_match_task_safe(req, tctx, cancel_all))
continue;
hlist_del_init(&req->hash_node);
if (cancel(req))
found = true;
}
return found;
}
int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
unsigned int issue_flags, struct hlist_head *list,
bool (*cancel)(struct io_kiocb *))
{
struct hlist_node *tmp;
struct io_kiocb *req;
int nr = 0;
io_ring_submit_lock(ctx, issue_flags);
hlist_for_each_entry_safe(req, tmp, list, hash_node) {
if (!io_cancel_req_match(req, cd))
continue;
if (cancel(req))
nr++;
if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
break;
}
io_ring_submit_unlock(ctx, issue_flags);
return nr ?: -ENOENT;
}

8
io_uring/cancel.h
View File

@ -24,6 +24,14 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);
bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);
bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
struct hlist_head *list, bool cancel_all,
bool (*cancel)(struct io_kiocb *));
int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
unsigned int issue_flags, struct hlist_head *list,
bool (*cancel)(struct io_kiocb *));
static inline bool io_cancel_match_sequence(struct io_kiocb *req, int sequence)
{
if (req->cancel_seq_set && sequence == req->work.cancel_seq)

2
io_uring/filetable.c
View File

@ -68,7 +68,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
if (slot_index >= ctx->file_table.data.nr)
return -EINVAL;
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
if (!node)
return -ENOMEM;

62
io_uring/futex.c
View File

@ -44,30 +44,28 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)
io_alloc_cache_free(&ctx->futex_cache, kfree);
}
static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
{
req->async_data = NULL;
hlist_del_init(&req->hash_node);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
}
static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_futex_data *ifd = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
io_tw_lock(ctx, ts);
if (!io_alloc_cache_put(&ctx->futex_cache, ifd))
kfree(ifd);
__io_futex_complete(req, ts);
io_tw_lock(ctx, tw);
io_cache_free(&ctx->futex_cache, req->async_data);
__io_futex_complete(req, tw);
}
static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv = req->async_data;
io_tw_lock(req->ctx, ts);
io_tw_lock(req->ctx, tw);
if (!iof->futexv_unqueued) {
int res;
@ -79,7 +77,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
kfree(req->async_data);
req->flags &= ~REQ_F_ASYNC_DATA;
__io_futex_complete(req, ts);
__io_futex_complete(req, tw);
}
static bool io_futexv_claim(struct io_futex *iof)
@ -90,7 +88,7 @@ static bool io_futexv_claim(struct io_futex *iof)
return true;
}
static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
static bool __io_futex_cancel(struct io_kiocb *req)
{
/* futex wake already done or in progress */
if (req->opcode == IORING_OP_FUTEX_WAIT) {
@ -116,49 +114,13 @@ static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
unsigned int issue_flags)
{
struct hlist_node *tmp;
struct io_kiocb *req;
int nr = 0;
if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED))
return -ENOENT;
io_ring_submit_lock(ctx, issue_flags);
hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
if (req->cqe.user_data != cd->data &&
!(cd->flags & IORING_ASYNC_CANCEL_ANY))
continue;
if (__io_futex_cancel(ctx, req))
nr++;
if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
break;
}
io_ring_submit_unlock(ctx, issue_flags);
if (nr)
return nr;
return -ENOENT;
return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel);
}
bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
bool cancel_all)
{
struct hlist_node *tmp;
struct io_kiocb *req;
bool found = false;
lockdep_assert_held(&ctx->uring_lock);
hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
if (!io_match_task_safe(req, tctx, cancel_all))
continue;
hlist_del_init(&req->hash_node);
__io_futex_cancel(ctx, req);
found = true;
}
return found;
return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel);
}
int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

230
io_uring/io-wq.c
View File

@ -30,7 +30,6 @@ enum {
IO_WORKER_F_UP = 0, /* up and active */
IO_WORKER_F_RUNNING = 1, /* account as running */
IO_WORKER_F_FREE = 2, /* worker on free list */
IO_WORKER_F_BOUND = 3, /* is doing bounded work */
};
enum {
@ -46,12 +45,12 @@ enum {
*/
struct io_worker {
refcount_t ref;
int create_index;
unsigned long flags;
struct hlist_nulls_node nulls_node;
struct list_head all_list;
struct task_struct *task;
struct io_wq *wq;
struct io_wq_acct *acct;
struct io_wq_work *cur_work;
raw_spinlock_t lock;
@ -77,10 +76,27 @@ struct io_worker {
#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER)
struct io_wq_acct {
/**
* Protects access to the worker lists.
*/
raw_spinlock_t workers_lock;
unsigned nr_workers;
unsigned max_workers;
int index;
atomic_t nr_running;
/**
* The list of free workers. Protected by #workers_lock
* (write) and RCU (read).
*/
struct hlist_nulls_head free_list;
/**
* The list of all workers. Protected by #workers_lock
* (write) and RCU (read).
*/
struct list_head all_list;
raw_spinlock_t lock;
struct io_wq_work_list work_list;
unsigned long flags;
@ -112,12 +128,6 @@ struct io_wq {
struct io_wq_acct acct[IO_WQ_ACCT_NR];
/* lock protects access to elements below */
raw_spinlock_t lock;
struct hlist_nulls_head free_list;
struct list_head all_list;
struct wait_queue_entry wait;
struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
@ -135,7 +145,7 @@ struct io_cb_cancel_data {
bool cancel_all;
};
static bool create_io_worker(struct io_wq *wq, int index);
static bool create_io_worker(struct io_wq *wq, struct io_wq_acct *acct);
static void io_wq_dec_running(struct io_worker *worker);
static bool io_acct_cancel_pending_work(struct io_wq *wq,
struct io_wq_acct *acct,
@ -160,14 +170,14 @@ static inline struct io_wq_acct *io_get_acct(struct io_wq *wq, bool bound)
}
static inline struct io_wq_acct *io_work_get_acct(struct io_wq *wq,
struct io_wq_work *work)
unsigned int work_flags)
{
return io_get_acct(wq, !(atomic_read(&work->flags) & IO_WQ_WORK_UNBOUND));
return io_get_acct(wq, !(work_flags & IO_WQ_WORK_UNBOUND));
}
static inline struct io_wq_acct *io_wq_get_acct(struct io_worker *worker)
{
return io_get_acct(worker->wq, test_bit(IO_WORKER_F_BOUND, &worker->flags));
return worker->acct;
}
static void io_worker_ref_put(struct io_wq *wq)
@ -192,9 +202,9 @@ static void io_worker_cancel_cb(struct io_worker *worker)
struct io_wq *wq = worker->wq;
atomic_dec(&acct->nr_running);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
acct->nr_workers--;
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
io_worker_ref_put(wq);
clear_bit_unlock(0, &worker->create_state);
io_worker_release(worker);
@ -213,6 +223,7 @@ static bool io_task_worker_match(struct callback_head *cb, void *data)
static void io_worker_exit(struct io_worker *worker)
{
struct io_wq *wq = worker->wq;
struct io_wq_acct *acct = io_wq_get_acct(worker);
while (1) {
struct callback_head *cb = task_work_cancel_match(wq->task,
@ -226,11 +237,11 @@ static void io_worker_exit(struct io_worker *worker)
io_worker_release(worker);
wait_for_completion(&worker->ref_done);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
if (test_bit(IO_WORKER_F_FREE, &worker->flags))
hlist_nulls_del_rcu(&worker->nulls_node);
list_del_rcu(&worker->all_list);
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
io_wq_dec_running(worker);
/*
* this worker is a goner, clear ->worker_private to avoid any
@ -269,8 +280,7 @@ static inline bool io_acct_run_queue(struct io_wq_acct *acct)
* Check head of free list for an available worker. If one isn't available,
* caller must create one.
*/
static bool io_wq_activate_free_worker(struct io_wq *wq,
struct io_wq_acct *acct)
static bool io_acct_activate_free_worker(struct io_wq_acct *acct)
__must_hold(RCU)
{
struct hlist_nulls_node *n;
@ -281,13 +291,9 @@ static bool io_wq_activate_free_worker(struct io_wq *wq,
* activate. If a given worker is on the free_list but in the process
* of exiting, keep trying.
*/
hlist_nulls_for_each_entry_rcu(worker, n, &wq->free_list, nulls_node) {
hlist_nulls_for_each_entry_rcu(worker, n, &acct->free_list, nulls_node) {
if (!io_worker_get(worker))
continue;
if (io_wq_get_acct(worker) != acct) {
io_worker_release(worker);
continue;
}
/*
* If the worker is already running, it's either already
* starting work or finishing work. In either case, if it does
@ -314,16 +320,16 @@ static bool io_wq_create_worker(struct io_wq *wq, struct io_wq_acct *acct)
if (unlikely(!acct->max_workers))
pr_warn_once("io-wq is not configured for unbound workers");
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
if (acct->nr_workers >= acct->max_workers) {
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
return true;
}
acct->nr_workers++;
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
atomic_inc(&acct->nr_running);
atomic_inc(&wq->worker_refs);
return create_io_worker(wq, acct->index);
return create_io_worker(wq, acct);
}
static void io_wq_inc_running(struct io_worker *worker)
@ -343,16 +349,16 @@ static void create_worker_cb(struct callback_head *cb)
worker = container_of(cb, struct io_worker, create_work);
wq = worker->wq;
acct = &wq->acct[worker->create_index];
raw_spin_lock(&wq->lock);
acct = worker->acct;
raw_spin_lock(&acct->workers_lock);
if (acct->nr_workers < acct->max_workers) {
acct->nr_workers++;
do_create = true;
}
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
if (do_create) {
create_io_worker(wq, worker->create_index);
create_io_worker(wq, acct);
} else {
atomic_dec(&acct->nr_running);
io_worker_ref_put(wq);
@ -384,7 +390,6 @@ static bool io_queue_worker_create(struct io_worker *worker,
atomic_inc(&wq->worker_refs);
init_task_work(&worker->create_work, func);
worker->create_index = acct->index;
if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) {
/*
* EXIT may have been set after checking it above, check after
@ -430,31 +435,36 @@ static void io_wq_dec_running(struct io_worker *worker)
* Worker will start processing some work. Move it to the busy list, if
* it's currently on the freelist
*/
static void __io_worker_busy(struct io_wq *wq, struct io_worker *worker)
static void __io_worker_busy(struct io_wq_acct *acct, struct io_worker *worker)
{
if (test_bit(IO_WORKER_F_FREE, &worker->flags)) {
clear_bit(IO_WORKER_F_FREE, &worker->flags);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
hlist_nulls_del_init_rcu(&worker->nulls_node);
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
}
}
/*
* No work, worker going to sleep. Move to freelist.
*/
static void __io_worker_idle(struct io_wq *wq, struct io_worker *worker)
__must_hold(wq->lock)
static void __io_worker_idle(struct io_wq_acct *acct, struct io_worker *worker)
__must_hold(acct->workers_lock)
{
if (!test_bit(IO_WORKER_F_FREE, &worker->flags)) {
set_bit(IO_WORKER_F_FREE, &worker->flags);
hlist_nulls_add_head_rcu(&worker->nulls_node, &wq->free_list);
hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list);
}
}
static inline unsigned int __io_get_work_hash(unsigned int work_flags)
{
return work_flags >> IO_WQ_HASH_SHIFT;
}
static inline unsigned int io_get_work_hash(struct io_wq_work *work)
{
return atomic_read(&work->flags) >> IO_WQ_HASH_SHIFT;
return __io_get_work_hash(atomic_read(&work->flags));
}
static bool io_wait_on_hash(struct io_wq *wq, unsigned int hash)
@ -475,26 +485,27 @@ static bool io_wait_on_hash(struct io_wq *wq, unsigned int hash)
}
static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
struct io_worker *worker)
struct io_wq *wq)
__must_hold(acct->lock)
{
struct io_wq_work_node *node, *prev;
struct io_wq_work *work, *tail;
unsigned int stall_hash = -1U;
struct io_wq *wq = worker->wq;
wq_list_for_each(node, prev, &acct->work_list) {
unsigned int work_flags;
unsigned int hash;
work = container_of(node, struct io_wq_work, list);
/* not hashed, can run anytime */
if (!io_wq_is_hashed(work)) {
work_flags = atomic_read(&work->flags);
if (!__io_wq_is_hashed(work_flags)) {
wq_list_del(&acct->work_list, node, prev);
return work;
}
hash = io_get_work_hash(work);
hash = __io_get_work_hash(work_flags);
/* all items with this hash lie in [work, tail] */
tail = wq->hash_tail[hash];
@ -564,7 +575,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
* can't make progress, any work completion or insertion will
* clear the stalled flag.
*/
work = io_get_next_work(acct, worker);
work = io_get_next_work(acct, wq);
if (work) {
/*
* Make sure cancelation can find this, even before
@ -583,7 +594,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
if (!work)
break;
__io_worker_busy(wq, worker);
__io_worker_busy(acct, worker);
io_assign_current_work(worker, work);
__set_current_state(TASK_RUNNING);
@ -591,12 +602,15 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
/* handle a whole dependent link */
do {
struct io_wq_work *next_hashed, *linked;
unsigned int hash = io_get_work_hash(work);
unsigned int work_flags = atomic_read(&work->flags);
unsigned int hash = __io_wq_is_hashed(work_flags)
? __io_get_work_hash(work_flags)
: -1U;
next_hashed = wq_next_work(work);
if (do_kill &&
(atomic_read(&work->flags) & IO_WQ_WORK_UNBOUND))
(work_flags & IO_WQ_WORK_UNBOUND))
atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
wq->do_work(work);
io_assign_current_work(worker, NULL);
@ -654,20 +668,20 @@ static int io_wq_worker(void *data)
while (io_acct_run_queue(acct))
io_worker_handle_work(acct, worker);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
/*
* Last sleep timed out. Exit if we're not the last worker,
* or if someone modified our affinity.
*/
if (last_timeout && (exit_mask || acct->nr_workers > 1)) {
acct->nr_workers--;
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
__set_current_state(TASK_RUNNING);
break;
}
last_timeout = false;
__io_worker_idle(wq, worker);
raw_spin_unlock(&wq->lock);
__io_worker_idle(acct, worker);
raw_spin_unlock(&acct->workers_lock);
if (io_run_task_work())
continue;
ret = schedule_timeout(WORKER_IDLE_TIMEOUT);
@ -728,18 +742,18 @@ void io_wq_worker_sleeping(struct task_struct *tsk)
io_wq_dec_running(worker);
}
static void io_init_new_worker(struct io_wq *wq, struct io_worker *worker,
static void io_init_new_worker(struct io_wq *wq, struct io_wq_acct *acct, struct io_worker *worker,
struct task_struct *tsk)
{
tsk->worker_private = worker;
worker->task = tsk;
set_cpus_allowed_ptr(tsk, wq->cpu_mask);
raw_spin_lock(&wq->lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &wq->free_list);
list_add_tail_rcu(&worker->all_list, &wq->all_list);
raw_spin_lock(&acct->workers_lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list);
list_add_tail_rcu(&worker->all_list, &acct->all_list);
set_bit(IO_WORKER_F_FREE, &worker->flags);
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
wake_up_new_task(tsk);
}
@ -787,20 +801,20 @@ static void create_worker_cont(struct callback_head *cb)
struct io_worker *worker;
struct task_struct *tsk;
struct io_wq *wq;
struct io_wq_acct *acct;
worker = container_of(cb, struct io_worker, create_work);
clear_bit_unlock(0, &worker->create_state);
wq = worker->wq;
acct = io_wq_get_acct(worker);
tsk = create_io_thread(io_wq_worker, worker, NUMA_NO_NODE);
if (!IS_ERR(tsk)) {
io_init_new_worker(wq, worker, tsk);
io_init_new_worker(wq, acct, worker, tsk);
io_worker_release(worker);
return;
} else if (!io_should_retry_thread(worker, PTR_ERR(tsk))) {
struct io_wq_acct *acct = io_wq_get_acct(worker);
atomic_dec(&acct->nr_running);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
acct->nr_workers--;
if (!acct->nr_workers) {
struct io_cb_cancel_data match = {
@ -808,11 +822,11 @@ static void create_worker_cont(struct callback_head *cb)
.cancel_all = true,
};
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
while (io_acct_cancel_pending_work(wq, acct, &match))
;
} else {
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
}
io_worker_ref_put(wq);
kfree(worker);
@ -834,9 +848,8 @@ static void io_workqueue_create(struct work_struct *work)
kfree(worker);
}
static bool create_io_worker(struct io_wq *wq, int index)
static bool create_io_worker(struct io_wq *wq, struct io_wq_acct *acct)
{
struct io_wq_acct *acct = &wq->acct[index];
struct io_worker *worker;
struct task_struct *tsk;
@ -846,24 +859,22 @@ static bool create_io_worker(struct io_wq *wq, int index)
if (!worker) {
fail:
atomic_dec(&acct->nr_running);
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
acct->nr_workers--;
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
io_worker_ref_put(wq);
return false;
}
refcount_set(&worker->ref, 1);
worker->wq = wq;
worker->acct = acct;
raw_spin_lock_init(&worker->lock);
init_completion(&worker->ref_done);
if (index == IO_WQ_ACCT_BOUND)
set_bit(IO_WORKER_F_BOUND, &worker->flags);
tsk = create_io_thread(io_wq_worker, worker, NUMA_NO_NODE);
if (!IS_ERR(tsk)) {
io_init_new_worker(wq, worker, tsk);
io_init_new_worker(wq, acct, worker, tsk);
} else if (!io_should_retry_thread(worker, PTR_ERR(tsk))) {
kfree(worker);
goto fail;
@ -879,14 +890,14 @@ static bool create_io_worker(struct io_wq *wq, int index)
* Iterate the passed in list and call the specific function for each
* worker that isn't exiting
*/
static bool io_wq_for_each_worker(struct io_wq *wq,
bool (*func)(struct io_worker *, void *),
void *data)
static bool io_acct_for_each_worker(struct io_wq_acct *acct,
bool (*func)(struct io_worker *, void *),
void *data)
{
struct io_worker *worker;
bool ret = false;
list_for_each_entry_rcu(worker, &wq->all_list, all_list) {
list_for_each_entry_rcu(worker, &acct->all_list, all_list) {
if (io_worker_get(worker)) {
/* no task if node is/was offline */
if (worker->task)
@ -900,6 +911,18 @@ static bool io_wq_for_each_worker(struct io_wq *wq,
return ret;
}
static bool io_wq_for_each_worker(struct io_wq *wq,
bool (*func)(struct io_worker *, void *),
void *data)
{
for (int i = 0; i < IO_WQ_ACCT_NR; i++) {
if (!io_acct_for_each_worker(&wq->acct[i], func, data))
return false;
}
return true;
}
static bool io_wq_worker_wake(struct io_worker *worker, void *data)
{
__set_notify_signal(worker->task);
@ -916,19 +939,19 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq)
} while (work);
}
static void io_wq_insert_work(struct io_wq *wq, struct io_wq_work *work)
static void io_wq_insert_work(struct io_wq *wq, struct io_wq_acct *acct,
struct io_wq_work *work, unsigned int work_flags)
{
struct io_wq_acct *acct = io_work_get_acct(wq, work);
unsigned int hash;
struct io_wq_work *tail;
if (!io_wq_is_hashed(work)) {
if (!__io_wq_is_hashed(work_flags)) {
append:
wq_list_add_tail(&work->list, &acct->work_list);
return;
}
hash = io_get_work_hash(work);
hash = __io_get_work_hash(work_flags);
tail = wq->hash_tail[hash];
wq->hash_tail[hash] = work;
if (!tail)
@ -944,8 +967,8 @@ static bool io_wq_work_match_item(struct io_wq_work *work, void *data)
void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
{
struct io_wq_acct *acct = io_work_get_acct(wq, work);
unsigned int work_flags = atomic_read(&work->flags);
struct io_wq_acct *acct = io_work_get_acct(wq, work_flags);
struct io_cb_cancel_data match = {
.fn = io_wq_work_match_item,
.data = work,
@ -964,12 +987,12 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
}
raw_spin_lock(&acct->lock);
io_wq_insert_work(wq, work);
io_wq_insert_work(wq, acct, work, work_flags);
clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
raw_spin_unlock(&acct->lock);
rcu_read_lock();
do_create = !io_wq_activate_free_worker(wq, acct);
do_create = !io_acct_activate_free_worker(acct);
rcu_read_unlock();
if (do_create && ((work_flags & IO_WQ_WORK_CONCURRENT) ||
@ -980,12 +1003,12 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
if (likely(did_create))
return;
raw_spin_lock(&wq->lock);
raw_spin_lock(&acct->workers_lock);
if (acct->nr_workers) {
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
return;
}
raw_spin_unlock(&wq->lock);
raw_spin_unlock(&acct->workers_lock);
/* fatal condition, failed to create the first worker */
io_acct_cancel_pending_work(wq, acct, &match);
@ -1034,10 +1057,10 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
}
static inline void io_wq_remove_pending(struct io_wq *wq,
struct io_wq_acct *acct,
struct io_wq_work *work,
struct io_wq_work_node *prev)
{
struct io_wq_acct *acct = io_work_get_acct(wq, work);
unsigned int hash = io_get_work_hash(work);
struct io_wq_work *prev_work = NULL;
@ -1064,7 +1087,7 @@ static bool io_acct_cancel_pending_work(struct io_wq *wq,
work = container_of(node, struct io_wq_work, list);
if (!match->fn(work, match->data))
continue;
io_wq_remove_pending(wq, work, prev);
io_wq_remove_pending(wq, acct, work, prev);
raw_spin_unlock(&acct->lock);
io_run_cancel(work, wq);
match->nr_pending++;
@ -1092,11 +1115,22 @@ static void io_wq_cancel_pending_work(struct io_wq *wq,
}
}
static void io_acct_cancel_running_work(struct io_wq_acct *acct,
struct io_cb_cancel_data *match)
{
raw_spin_lock(&acct->workers_lock);
io_acct_for_each_worker(acct, io_wq_worker_cancel, match);
raw_spin_unlock(&acct->workers_lock);
}
static void io_wq_cancel_running_work(struct io_wq *wq,
struct io_cb_cancel_data *match)
{
rcu_read_lock();
io_wq_for_each_worker(wq, io_wq_worker_cancel, match);
for (int i = 0; i < IO_WQ_ACCT_NR; i++)
io_acct_cancel_running_work(&wq->acct[i], match);
rcu_read_unlock();
}
@ -1119,16 +1153,14 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
* as an indication that we attempt to signal cancellation. The
* completion will run normally in this case.
*
* Do both of these while holding the wq->lock, to ensure that
* Do both of these while holding the acct->workers_lock, to ensure that
* we'll find a work item regardless of state.
*/
io_wq_cancel_pending_work(wq, &match);
if (match.nr_pending && !match.cancel_all)
return IO_WQ_CANCEL_OK;
raw_spin_lock(&wq->lock);
io_wq_cancel_running_work(wq, &match);
raw_spin_unlock(&wq->lock);
if (match.nr_running && !match.cancel_all)
return IO_WQ_CANCEL_RUNNING;
@ -1152,7 +1184,7 @@ static int io_wq_hash_wake(struct wait_queue_entry *wait, unsigned mode,
struct io_wq_acct *acct = &wq->acct[i];
if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))
io_wq_activate_free_worker(wq, acct);
io_acct_activate_free_worker(acct);
}
rcu_read_unlock();
return 1;
@ -1190,16 +1222,16 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
for (i = 0; i < IO_WQ_ACCT_NR; i++) {
struct io_wq_acct *acct = &wq->acct[i];
acct->index = i;
atomic_set(&acct->nr_running, 0);
raw_spin_lock_init(&acct->workers_lock);
INIT_HLIST_NULLS_HEAD(&acct->free_list, 0);
INIT_LIST_HEAD(&acct->all_list);
INIT_WQ_LIST(&acct->work_list);
raw_spin_lock_init(&acct->lock);
}
raw_spin_lock_init(&wq->lock);
INIT_HLIST_NULLS_HEAD(&wq->free_list, 0);
INIT_LIST_HEAD(&wq->all_list);
wq->task = get_task_struct(data->task);
atomic_set(&wq->worker_refs, 1);
init_completion(&wq->worker_done);
@ -1385,14 +1417,14 @@ int io_wq_max_workers(struct io_wq *wq, int *new_count)
rcu_read_lock();
raw_spin_lock(&wq->lock);
for (i = 0; i < IO_WQ_ACCT_NR; i++) {
acct = &wq->acct[i];
raw_spin_lock(&acct->workers_lock);
prev[i] = max_t(int, acct->max_workers, prev[i]);
if (new_count[i])
acct->max_workers = new_count[i];
raw_spin_unlock(&acct->workers_lock);
}
raw_spin_unlock(&wq->lock);
rcu_read_unlock();
for (i = 0; i < IO_WQ_ACCT_NR; i++)

7
io_uring/io-wq.h
View File

@ -54,9 +54,14 @@ int io_wq_cpu_affinity(struct io_uring_task *tctx, cpumask_var_t mask);
int io_wq_max_workers(struct io_wq *wq, int *new_count);
bool io_wq_worker_stopped(void);
static inline bool __io_wq_is_hashed(unsigned int work_flags)
{
return work_flags & IO_WQ_WORK_HASHED;
}
static inline bool io_wq_is_hashed(struct io_wq_work *work)
{
return atomic_read(&work->flags) & IO_WQ_WORK_HASHED;
return __io_wq_is_hashed(atomic_read(&work->flags));
}
typedef bool (work_cancel_fn)(struct io_wq_work *, void *);

217
io_uring/io_uring.c
View File

@ -110,11 +110,13 @@
#define SQE_VALID_FLAGS (SQE_COMMON_FLAGS | IOSQE_BUFFER_SELECT | \
IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS)
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \
REQ_F_ASYNC_DATA)
#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\
#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | IO_REQ_LINK_FLAGS | \
REQ_F_REISSUE | IO_REQ_CLEAN_FLAGS)
#define IO_TCTX_REFS_CACHE_NR (1U << 10)
@ -131,7 +133,6 @@ struct io_defer_entry {
/* requests with any of those set should undergo io_disarm_next() */
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
/*
* No waiters. It's larger than any valid value of the tw counter
@ -254,7 +255,7 @@ static __cold void io_fallback_req_func(struct work_struct *work)
percpu_ref_get(&ctx->refs);
mutex_lock(&ctx->uring_lock);
llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
req->io_task_work.func(req, &ts);
req->io_task_work.func(req, ts);
io_submit_flush_completions(ctx);
mutex_unlock(&ctx->uring_lock);
percpu_ref_put(&ctx->refs);
@ -282,6 +283,17 @@ static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits)
return 0;
}
static void io_free_alloc_caches(struct io_ring_ctx *ctx)
{
io_alloc_cache_free(&ctx->apoll_cache, kfree);
io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
io_alloc_cache_free(&ctx->uring_cache, kfree);
io_alloc_cache_free(&ctx->msg_cache, kfree);
io_futex_cache_free(ctx);
io_rsrc_cache_free(ctx);
}
static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
struct io_ring_ctx *ctx;
@ -313,7 +325,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
init_waitqueue_head(&ctx->sqo_sq_wait);
INIT_LIST_HEAD(&ctx->sqd_list);
INIT_LIST_HEAD(&ctx->cq_overflow_list);
INIT_LIST_HEAD(&ctx->io_buffers_cache);
ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
sizeof(struct async_poll), 0);
ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
@ -328,6 +339,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
ret |= io_alloc_cache_init(&ctx->msg_cache, IO_ALLOC_CACHE_MAX,
sizeof(struct io_kiocb), 0);
ret |= io_futex_cache_init(ctx);
ret |= io_rsrc_cache_init(ctx);
if (ret)
goto free_ref;
init_completion(&ctx->ref_comp);
@ -338,7 +350,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
spin_lock_init(&ctx->completion_lock);
raw_spin_lock_init(&ctx->timeout_lock);
INIT_WQ_LIST(&ctx->iopoll_list);
INIT_LIST_HEAD(&ctx->io_buffers_comp);
INIT_LIST_HEAD(&ctx->defer_list);
INIT_LIST_HEAD(&ctx->timeout_list);
INIT_LIST_HEAD(&ctx->ltimeout_list);
@ -360,12 +371,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
free_ref:
percpu_ref_exit(&ctx->refs);
err:
io_alloc_cache_free(&ctx->apoll_cache, kfree);
io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
io_alloc_cache_free(&ctx->uring_cache, kfree);
io_alloc_cache_free(&ctx->msg_cache, kfree);
io_futex_cache_free(ctx);
io_free_alloc_caches(ctx);
kvfree(ctx->cancel_table.hbs);
xa_destroy(&ctx->io_bl_xa);
kfree(ctx);
@ -393,11 +399,8 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
static void io_clean_op(struct io_kiocb *req)
{
if (req->flags & REQ_F_BUFFER_SELECTED) {
spin_lock(&req->ctx->completion_lock);
io_kbuf_drop(req);
spin_unlock(&req->ctx->completion_lock);
}
if (unlikely(req->flags & REQ_F_BUFFER_SELECTED))
io_kbuf_drop_legacy(req);
if (req->flags & REQ_F_NEED_CLEANUP) {
const struct io_cold_def *def = &io_cold_defs[req->opcode];
@ -542,7 +545,7 @@ static void io_queue_iowq(struct io_kiocb *req)
io_queue_linked_timeout(link);
}
static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)
static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
{
io_queue_iowq(req);
}
@ -899,7 +902,7 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
* Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
* the submitter task context, IOPOLL protects with uring_lock.
*/
if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL)) {
if (ctx->lockless_cq) {
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
return;
@ -1021,7 +1024,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
return nxt;
}
static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)
static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)
{
if (!ctx)
return;
@ -1051,24 +1054,24 @@ struct llist_node *io_handle_tw_list(struct llist_node *node,
io_task_work.node);
if (req->ctx != ctx) {
ctx_flush_and_put(ctx, &ts);
ctx_flush_and_put(ctx, ts);
ctx = req->ctx;
mutex_lock(&ctx->uring_lock);
percpu_ref_get(&ctx->refs);
}
INDIRECT_CALL_2(req->io_task_work.func,
io_poll_task_func, io_req_rw_complete,
req, &ts);
req, ts);
node = next;
(*count)++;
if (unlikely(need_resched())) {
ctx_flush_and_put(ctx, &ts);
ctx_flush_and_put(ctx, ts);
ctx = NULL;
cond_resched();
}
} while (node && *count < max_entries);
ctx_flush_and_put(ctx, &ts);
ctx_flush_and_put(ctx, ts);
return node;
}
@ -1157,7 +1160,7 @@ static inline void io_req_local_work_add(struct io_kiocb *req,
* We don't know how many reuqests is there in the link and whether
* they can even be queued lazily, fall back to non-lazy.
*/
if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK))
if (req->flags & IO_REQ_LINK_FLAGS)
flags &= ~IOU_F_TWQ_LAZY_WAKE;
guard(rcu)();
@ -1276,7 +1279,7 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
}
static int __io_run_local_work_loop(struct llist_node **node,
struct io_tw_state *ts,
io_tw_token_t tw,
int events)
{
int ret = 0;
@ -1287,7 +1290,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
io_task_work.node);
INDIRECT_CALL_2(req->io_task_work.func,
io_poll_task_func, io_req_rw_complete,
req, ts);
req, tw);
*node = next;
if (++ret >= events)
break;
@ -1296,7 +1299,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
return ret;
}
static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
int min_events, int max_events)
{
struct llist_node *node;
@ -1309,7 +1312,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
again:
min_events -= ret;
ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events);
ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
if (ctx->retry_llist.first)
goto retry_done;
@ -1318,7 +1321,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
* running the pending items.
*/
node = llist_reverse_order(llist_del_all(&ctx->work_llist));
ret += __io_run_local_work_loop(&node, ts, max_events - ret);
ret += __io_run_local_work_loop(&node, tw, max_events - ret);
ctx->retry_llist.first = node;
loops++;
@ -1340,7 +1343,7 @@ static inline int io_run_local_work_locked(struct io_ring_ctx *ctx,
if (!io_local_work_pending(ctx))
return 0;
return __io_run_local_work(ctx, &ts, min_events,
return __io_run_local_work(ctx, ts, min_events,
max(IO_LOCAL_TW_DEFAULT_MAX, min_events));
}
@ -1351,20 +1354,20 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
int ret;
mutex_lock(&ctx->uring_lock);
ret = __io_run_local_work(ctx, &ts, min_events, max_events);
ret = __io_run_local_work(ctx, ts, min_events, max_events);
mutex_unlock(&ctx->uring_lock);
return ret;
}
static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)
static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
{
io_tw_lock(req->ctx, ts);
io_tw_lock(req->ctx, tw);
io_req_defer_failed(req, req->cqe.res);
}
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
{
io_tw_lock(req->ctx, ts);
io_tw_lock(req->ctx, tw);
if (unlikely(io_should_terminate_tw()))
io_req_defer_failed(req, -EFAULT);
else if (req->flags & REQ_F_FORCE_ASYNC)
@ -1419,8 +1422,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx,
if (apoll->double_poll)
kfree(apoll->double_poll);
if (!io_alloc_cache_put(&ctx->apoll_cache, apoll))
kfree(apoll);
io_cache_free(&ctx->apoll_cache, apoll);
req->flags &= ~REQ_F_POLLED;
}
if (req->flags & IO_REQ_LINK_FLAGS)
@ -1582,7 +1584,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
return 0;
}
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
{
io_req_complete_defer(req);
}
@ -1719,15 +1721,13 @@ static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
return !!req->file;
}
static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
static inline int __io_issue_sqe(struct io_kiocb *req,
unsigned int issue_flags,
const struct io_issue_def *def)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
const struct cred *creds = NULL;
int ret;
if (unlikely(!io_assign_file(req, def, issue_flags)))
return -EBADF;
if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred()))
creds = override_creds(req->creds);
@ -1742,6 +1742,19 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
if (creds)
revert_creds(creds);
return ret;
}
static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
int ret;
if (unlikely(!io_assign_file(req, def, issue_flags)))
return -EBADF;
ret = __io_issue_sqe(req, issue_flags, def);
if (ret == IOU_OK) {
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
io_req_complete_defer(req);
@ -1762,11 +1775,26 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
return ret;
}
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
{
io_tw_lock(req->ctx, ts);
return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
IO_URING_F_COMPLETE_DEFER);
const unsigned int issue_flags = IO_URING_F_NONBLOCK |
IO_URING_F_MULTISHOT |
IO_URING_F_COMPLETE_DEFER;
int ret;
io_tw_lock(req->ctx, tw);
WARN_ON_ONCE(!req->file);
if (WARN_ON_ONCE(req->ctx->flags & IORING_SETUP_IOPOLL))
return -EFAULT;
ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]);
WARN_ON_ONCE(ret == IOU_OK);
if (ret == IOU_ISSUE_SKIP_COMPLETE)
ret = 0;
return ret;
}
struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
@ -1996,9 +2024,8 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx,
return true;
}
static void io_init_req_drain(struct io_kiocb *req)
static void io_init_drain(struct io_ring_ctx *ctx)
{
struct io_ring_ctx *ctx = req->ctx;
struct io_kiocb *head = ctx->submit_state.link.head;
ctx->drain_active = true;
@ -2062,7 +2089,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
if (sqe_flags & IOSQE_IO_DRAIN) {
if (ctx->drain_disabled)
return io_init_fail_req(req, -EOPNOTSUPP);
io_init_req_drain(req);
io_init_drain(ctx);
}
}
if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
@ -2704,12 +2731,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_sqe_files_unregister(ctx);
io_cqring_overflow_kill(ctx);
io_eventfd_unregister(ctx);
io_alloc_cache_free(&ctx->apoll_cache, kfree);
io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
io_alloc_cache_free(&ctx->uring_cache, kfree);
io_alloc_cache_free(&ctx->msg_cache, kfree);
io_futex_cache_free(ctx);
io_free_alloc_caches(ctx);
io_destroy_buffers(ctx);
io_free_region(ctx, &ctx->param_region);
mutex_unlock(&ctx->uring_lock);
@ -3537,6 +3559,44 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
O_RDWR | O_CLOEXEC, NULL);
}
static int io_uring_sanitise_params(struct io_uring_params *p)
{
unsigned flags = p->flags;
/* There is no way to mmap rings without a real fd */
if ((flags & IORING_SETUP_REGISTERED_FD_ONLY) &&
!(flags & IORING_SETUP_NO_MMAP))
return -EINVAL;
if (flags & IORING_SETUP_SQPOLL) {
/* IPI related flags don't make sense with SQPOLL */
if (flags & (IORING_SETUP_COOP_TASKRUN |
IORING_SETUP_TASKRUN_FLAG |
IORING_SETUP_DEFER_TASKRUN))
return -EINVAL;
}
if (flags & IORING_SETUP_TASKRUN_FLAG) {
if (!(flags & (IORING_SETUP_COOP_TASKRUN |
IORING_SETUP_DEFER_TASKRUN)))
return -EINVAL;
}
/* HYBRID_IOPOLL only valid with IOPOLL */
if ((flags & IORING_SETUP_HYBRID_IOPOLL) && !(flags & IORING_SETUP_IOPOLL))
return -EINVAL;
/*
* For DEFER_TASKRUN we require the completion task to be the same as
* the submission task. This implies that there is only one submitter.
*/
if ((flags & IORING_SETUP_DEFER_TASKRUN) &&
!(flags & IORING_SETUP_SINGLE_ISSUER))
return -EINVAL;
return 0;
}
int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
{
if (!entries)
@ -3547,10 +3607,6 @@ int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
entries = IORING_MAX_ENTRIES;
}
if ((p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
&& !(p->flags & IORING_SETUP_NO_MMAP))
return -EINVAL;
/*
* Use twice as many entries for the CQ ring. It's possible for the
* application to drive a higher depth than the size of the SQ ring,
@ -3612,6 +3668,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
struct file *file;
int ret;
ret = io_uring_sanitise_params(p);
if (ret)
return ret;
ret = io_uring_fill_params(entries, p);
if (unlikely(ret))
return ret;
@ -3659,37 +3719,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
* For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
* COOP_TASKRUN is set, then IPIs are never needed by the app.
*/
ret = -EINVAL;
if (ctx->flags & IORING_SETUP_SQPOLL) {
/* IPI related flags don't make sense with SQPOLL */
if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
IORING_SETUP_TASKRUN_FLAG |
IORING_SETUP_DEFER_TASKRUN))
goto err;
if (ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_COOP_TASKRUN))
ctx->notify_method = TWA_SIGNAL_NO_IPI;
} else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
ctx->notify_method = TWA_SIGNAL_NO_IPI;
} else {
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG &&
!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
goto err;
else
ctx->notify_method = TWA_SIGNAL;
}
/* HYBRID_IOPOLL only valid with IOPOLL */
if ((ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_HYBRID_IOPOLL)) ==
IORING_SETUP_HYBRID_IOPOLL)
goto err;
/*
* For DEFER_TASKRUN we require the completion task to be the same as the
* submission task. This implies that there is only one submitter, so enforce
* that.
*/
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN &&
!(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) {
goto err;
}
/*
* This is just grabbed for accounting purposes. When a process exits,
@ -3908,6 +3941,9 @@ static int __init io_uring_init(void)
io_uring_optable_init();
/* imu->dir is u8 */
BUILD_BUG_ON((IO_IMU_DEST | IO_IMU_SOURCE) > U8_MAX);
/*
* Allow user copy in the per-command field, which starts after the
* file in io_kiocb and until the opcode field. The openat2 handling
@ -3918,10 +3954,9 @@ static int __init io_uring_init(void)
req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,
SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
SLAB_TYPESAFE_BY_RCU);
io_buf_cachep = KMEM_CACHE(io_buffer,
SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT);
iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
BUG_ON(!iou_wq);
#ifdef CONFIG_SYSCTL
register_sysctl_init("kernel", kernel_io_uring_disabled_table);

14
io_uring/io_uring.h
View File

@ -90,9 +90,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
unsigned flags);
bool io_alloc_async_data(struct io_kiocb *req);
void io_req_task_queue(struct io_kiocb *req);
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
void io_req_task_queue_fail(struct io_kiocb *req, int ret);
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts);
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
void tctx_task_work(struct callback_head *cb);
@ -104,7 +104,7 @@ int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
int start, int end);
void io_req_queue_iowq(struct io_kiocb *req);
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@ -147,6 +147,11 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
#endif
}
static inline bool io_is_compat(struct io_ring_ctx *ctx)
{
return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->compat);
}
static inline void io_req_task_work_add(struct io_kiocb *req)
{
__io_req_task_work_add(req, 0);
@ -376,7 +381,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
return task_work_pending(current) || io_local_work_pending(ctx);
}
static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)
static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)
{
lockdep_assert_held(&ctx->uring_lock);
}
@ -418,7 +423,6 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
}
extern struct kmem_cache *req_cachep;
extern struct kmem_cache *io_buf_cachep;
static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
{

172
io_uring/kbuf.c
View File

@ -20,7 +20,8 @@
/* BIDs are addressed by a 16-bit field in a CQE */
#define MAX_BIDS_PER_BGID (1 << 16)
struct kmem_cache *io_buf_cachep;
/* Mapped buffer ring, return io_uring_buf from head */
#define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)]
struct io_provide_buf {
struct file *file;
@ -31,6 +32,34 @@ struct io_provide_buf {
__u16 bid;
};
bool io_kbuf_commit(struct io_kiocb *req,
struct io_buffer_list *bl, int len, int nr)
{
if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
return true;
req->flags &= ~REQ_F_BUFFERS_COMMIT;
if (unlikely(len < 0))
return true;
if (bl->flags & IOBL_INC) {
struct io_uring_buf *buf;
buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
if (WARN_ON_ONCE(len > buf->len))
len = buf->len;
buf->len -= len;
if (buf->len) {
buf->addr += len;
return false;
}
}
bl->head += nr;
return true;
}
static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
unsigned int bgid)
{
@ -52,6 +81,16 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx,
return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
}
void io_kbuf_drop_legacy(struct io_kiocb *req)
{
if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED)))
return;
req->buf_index = req->kbuf->bgid;
req->flags &= ~REQ_F_BUFFER_SELECTED;
kfree(req->kbuf);
req->kbuf = NULL;
}
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
{
struct io_ring_ctx *ctx = req->ctx;
@ -70,33 +109,6 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
return true;
}
void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags)
{
/*
* We can add this buffer back to two lists:
*
* 1) The io_buffers_cache list. This one is protected by the
* ctx->uring_lock. If we already hold this lock, add back to this
* list as we can grab it from issue as well.
* 2) The io_buffers_comp list. This one is protected by the
* ctx->completion_lock.
*
* We migrate buffers from the comp_list to the issue cache list
* when we need one.
*/
if (issue_flags & IO_URING_F_UNLOCKED) {
struct io_ring_ctx *ctx = req->ctx;
spin_lock(&ctx->completion_lock);
__io_put_kbuf_list(req, len, &ctx->io_buffers_comp);
spin_unlock(&ctx->completion_lock);
} else {
lockdep_assert_held(&req->ctx->uring_lock);
__io_put_kbuf_list(req, len, &req->ctx->io_buffers_cache);
}
}
static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
struct io_buffer_list *bl)
{
@ -342,6 +354,35 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg)
return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs);
}
static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
{
struct io_buffer_list *bl = req->buf_list;
bool ret = true;
if (bl) {
ret = io_kbuf_commit(req, bl, len, nr);
req->buf_index = bl->bgid;
}
req->flags &= ~REQ_F_BUFFER_RING;
return ret;
}
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)
{
unsigned int ret;
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) {
io_kbuf_drop_legacy(req);
return ret;
}
if (!__io_put_kbuf_ring(req, len, nbufs))
ret |= IORING_CQE_F_BUF_MORE;
return ret;
}
static int __io_remove_buffers(struct io_ring_ctx *ctx,
struct io_buffer_list *bl, unsigned nbufs)
{
@ -367,7 +408,9 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,
struct io_buffer *nxt;
nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
list_move(&nxt->list, &ctx->io_buffers_cache);
list_del(&nxt->list);
kfree(nxt);
if (++i == nbufs)
return i;
cond_resched();
@ -385,8 +428,6 @@ static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
void io_destroy_buffers(struct io_ring_ctx *ctx)
{
struct io_buffer_list *bl;
struct list_head *item, *tmp;
struct io_buffer *buf;
while (1) {
unsigned long index = 0;
@ -400,19 +441,6 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)
break;
io_put_bl(ctx, bl);
}
/*
* Move deferred locked entries to cache before pruning
*/
spin_lock(&ctx->completion_lock);
if (!list_empty(&ctx->io_buffers_comp))
list_splice_init(&ctx->io_buffers_comp, &ctx->io_buffers_cache);
spin_unlock(&ctx->completion_lock);
list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {
buf = list_entry(item, struct io_buffer, list);
kmem_cache_free(io_buf_cachep, buf);
}
}
static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
@ -501,53 +529,6 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
return 0;
}
#define IO_BUFFER_ALLOC_BATCH 64
static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
{
struct io_buffer *bufs[IO_BUFFER_ALLOC_BATCH];
int allocated;
/*
* Completions that don't happen inline (eg not under uring_lock) will
* add to ->io_buffers_comp. If we don't have any free buffers, check
* the completion list and splice those entries first.
*/
if (!list_empty_careful(&ctx->io_buffers_comp)) {
spin_lock(&ctx->completion_lock);
if (!list_empty(&ctx->io_buffers_comp)) {
list_splice_init(&ctx->io_buffers_comp,
&ctx->io_buffers_cache);
spin_unlock(&ctx->completion_lock);
return 0;
}
spin_unlock(&ctx->completion_lock);
}
/*
* No free buffers and no completion entries either. Allocate a new
* batch of buffer entries and add those to our freelist.
*/
allocated = kmem_cache_alloc_bulk(io_buf_cachep, GFP_KERNEL_ACCOUNT,
ARRAY_SIZE(bufs), (void **) bufs);
if (unlikely(!allocated)) {
/*
* Bulk alloc is all-or-nothing. If we fail to get a batch,
* retry single alloc to be on the safe side.
*/
bufs[0] = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
if (!bufs[0])
return -ENOMEM;
allocated = 1;
}
while (allocated)
list_add_tail(&bufs[--allocated]->list, &ctx->io_buffers_cache);
return 0;
}
static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
struct io_buffer_list *bl)
{
@ -556,12 +537,11 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
int i, bid = pbuf->bid;
for (i = 0; i < pbuf->nbufs; i++) {
if (list_empty(&ctx->io_buffers_cache) &&
io_refill_buffer_cache(ctx))
buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
if (!buf)
break;
buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
list);
list_move_tail(&buf->list, &bl->buf_list);
list_add_tail(&buf->list, &bl->buf_list);
buf->addr = addr;
buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
buf->bid = bid;

100
io_uring/kbuf.h
View File

@ -74,9 +74,12 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags);
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
void io_kbuf_drop_legacy(struct io_kiocb *req);
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs);
bool io_kbuf_commit(struct io_kiocb *req,
struct io_buffer_list *bl, int len, int nr);
struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
unsigned int bgid);
@ -116,100 +119,19 @@ static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
return false;
}
/* Mapped buffer ring, return io_uring_buf from head */
#define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)]
static inline bool io_kbuf_commit(struct io_kiocb *req,
struct io_buffer_list *bl, int len, int nr)
{
if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
return true;
req->flags &= ~REQ_F_BUFFERS_COMMIT;
if (unlikely(len < 0))
return true;
if (bl->flags & IOBL_INC) {
struct io_uring_buf *buf;
buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
if (WARN_ON_ONCE(len > buf->len))
len = buf->len;
buf->len -= len;
if (buf->len) {
buf->addr += len;
return false;
}
}
bl->head += nr;
return true;
}
static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
{
struct io_buffer_list *bl = req->buf_list;
bool ret = true;
if (bl) {
ret = io_kbuf_commit(req, bl, len, nr);
req->buf_index = bl->bgid;
}
req->flags &= ~REQ_F_BUFFER_RING;
return ret;
}
static inline void __io_put_kbuf_list(struct io_kiocb *req, int len,
struct list_head *list)
{
if (req->flags & REQ_F_BUFFER_RING) {
__io_put_kbuf_ring(req, len, 1);
} else {
req->buf_index = req->kbuf->bgid;
list_add(&req->kbuf->list, list);
req->flags &= ~REQ_F_BUFFER_SELECTED;
}
}
static inline void io_kbuf_drop(struct io_kiocb *req)
{
lockdep_assert_held(&req->ctx->completion_lock);
if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
return;
/* len == 0 is fine here, non-ring will always drop all of it */
__io_put_kbuf_list(req, 0, &req->ctx->io_buffers_comp);
}
static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
int nbufs, unsigned issue_flags)
{
unsigned int ret;
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
return 0;
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
if (req->flags & REQ_F_BUFFER_RING) {
if (!__io_put_kbuf_ring(req, len, nbufs))
ret |= IORING_CQE_F_BUF_MORE;
} else {
__io_put_kbuf(req, len, issue_flags);
}
return ret;
}
static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len,
unsigned issue_flags)
{
return __io_put_kbufs(req, len, 1, issue_flags);
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
return 0;
return __io_put_kbufs(req, len, 1);
}
static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len,
int nbufs, unsigned issue_flags)
{
return __io_put_kbufs(req, len, nbufs, issue_flags);
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
return 0;
return __io_put_kbufs(req, len, nbufs);
}
#endif

2
io_uring/msg_ring.c
View File

@ -71,7 +71,7 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
return target_ctx->task_complete;
}
static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts)
static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_ring_ctx *ctx = req->ctx;

257
io_uring/net.c
View File

@ -75,7 +75,7 @@ struct io_sr_msg {
u16 flags;
/* initialised and used only by !msg send variants */
u16 buf_group;
u16 buf_index;
bool retry;
void __user *msg_control;
/* used only for send zerocopy */
struct io_kiocb *notif;
@ -187,17 +187,15 @@ static inline void io_mshot_prep_retry(struct io_kiocb *req,
req->flags &= ~REQ_F_BL_EMPTY;
sr->done_io = 0;
sr->retry = false;
sr->len = 0; /* get from the provided buffer */
req->buf_index = sr->buf_group;
}
#ifdef CONFIG_COMPAT
static int io_compat_msg_copy_hdr(struct io_kiocb *req,
struct io_async_msghdr *iomsg,
struct compat_msghdr *msg, int ddir)
static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,
const struct iovec __user *uiov, unsigned uvec_seg,
int ddir)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct compat_iovec __user *uiov;
struct iovec *iov;
int ret, nr_segs;
@ -205,103 +203,108 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
nr_segs = iomsg->free_iov_nr;
iov = iomsg->free_iov;
} else {
iov = &iomsg->fast_iov;
nr_segs = 1;
iov = &iomsg->fast_iov;
}
ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,
&iomsg->msg.msg_iter, io_is_compat(req->ctx));
if (unlikely(ret < 0))
return ret;
io_net_vec_assign(req, iomsg, iov);
return 0;
}
static int io_compat_msg_copy_hdr(struct io_kiocb *req,
struct io_async_msghdr *iomsg,
struct compat_msghdr *msg, int ddir,
struct sockaddr __user **save_addr)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct compat_iovec __user *uiov;
int ret;
if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
return -EFAULT;
ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr);
if (ret)
return ret;
uiov = compat_ptr(msg->msg_iov);
if (req->flags & REQ_F_BUFFER_SELECT) {
compat_ssize_t clen;
if (msg->msg_iovlen == 0) {
sr->len = iov->iov_len = 0;
iov->iov_base = NULL;
sr->len = 0;
} else if (msg->msg_iovlen > 1) {
return -EINVAL;
} else {
if (!access_ok(uiov, sizeof(*uiov)))
struct compat_iovec tmp_iov;
if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))
return -EFAULT;
if (__get_user(clen, &uiov->iov_len))
return -EFAULT;
if (clen < 0)
return -EINVAL;
sr->len = clen;
sr->len = tmp_iov.iov_len;
}
return 0;
}
ret = __import_iovec(ddir, (struct iovec __user *)uiov, msg->msg_iovlen,
nr_segs, &iov, &iomsg->msg.msg_iter, true);
if (unlikely(ret < 0))
return ret;
io_net_vec_assign(req, iomsg, iov);
return 0;
return io_net_import_vec(req, iomsg, (struct iovec __user *)uiov,
msg->msg_iovlen, ddir);
}
#endif
static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
struct user_msghdr *msg, int ddir)
static int io_copy_msghdr_from_user(struct user_msghdr *msg,
struct user_msghdr __user *umsg)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct user_msghdr __user *umsg = sr->umsg;
struct iovec *iov;
int ret, nr_segs;
if (iomsg->free_iov) {
nr_segs = iomsg->free_iov_nr;
iov = iomsg->free_iov;
} else {
iov = &iomsg->fast_iov;
nr_segs = 1;
}
if (!user_access_begin(umsg, sizeof(*umsg)))
return -EFAULT;
ret = -EFAULT;
unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
user_access_end();
return 0;
ua_end:
user_access_end();
return -EFAULT;
}
static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
struct user_msghdr *msg, int ddir,
struct sockaddr __user **save_addr)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct user_msghdr __user *umsg = sr->umsg;
int ret;
ret = io_copy_msghdr_from_user(msg, umsg);
if (unlikely(ret))
return ret;
msg->msg_flags = 0;
ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
if (ret)
return ret;
if (req->flags & REQ_F_BUFFER_SELECT) {
if (msg->msg_iovlen == 0) {
sr->len = iov->iov_len = 0;
iov->iov_base = NULL;
sr->len = 0;
} else if (msg->msg_iovlen > 1) {
ret = -EINVAL;
goto ua_end;
return -EINVAL;
} else {
struct iovec __user *uiov = msg->msg_iov;
struct iovec tmp_iov;
/* we only need the length for provided buffers */
if (!access_ok(&uiov->iov_len, sizeof(uiov->iov_len)))
goto ua_end;
unsafe_get_user(iov->iov_len, &uiov->iov_len, ua_end);
sr->len = iov->iov_len;
if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))
return -EFAULT;
sr->len = tmp_iov.iov_len;
}
ret = 0;
ua_end:
user_access_end();
return ret;
return 0;
}
user_access_end();
ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, nr_segs,
&iov, &iomsg->msg.msg_iter, false);
if (unlikely(ret < 0))
return ret;
io_net_vec_assign(req, iomsg, iov);
return 0;
return io_net_import_vec(req, iomsg, msg->msg_iov, msg->msg_iovlen, ddir);
}
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
@ -314,26 +317,16 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,
iomsg->msg.msg_name = &iomsg->addr;
iomsg->msg.msg_iter.nr_segs = 0;
#ifdef CONFIG_COMPAT
if (unlikely(req->ctx->compat)) {
if (io_is_compat(req->ctx)) {
struct compat_msghdr cmsg;
ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_SOURCE);
if (unlikely(ret))
return ret;
ret = __get_compat_msghdr(&iomsg->msg, &cmsg, NULL);
ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_SOURCE,
NULL);
sr->msg_control = iomsg->msg.msg_control_user;
return ret;
}
#endif
ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_SOURCE);
if (unlikely(ret))
return ret;
ret = __copy_msghdr(&iomsg->msg, &msg, NULL);
ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_SOURCE, NULL);
/* save msg_control as sys_sendmsg() overwrites it */
sr->msg_control = iomsg->msg.msg_control_user;
return ret;
@ -387,14 +380,10 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data;
int ret;
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
ret = io_sendmsg_copy_hdr(req, kmsg);
if (!ret)
req->flags |= REQ_F_NEED_CLEANUP;
return ret;
return io_sendmsg_copy_hdr(req, kmsg);
}
#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)
@ -404,6 +393,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
sr->done_io = 0;
sr->retry = false;
if (req->opcode != IORING_OP_SEND) {
if (sqe->addr2 || sqe->file_index)
@ -427,10 +417,9 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
req->buf_list = NULL;
}
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
if (io_is_compat(req->ctx))
sr->msg_flags |= MSG_CMSG_COMPAT;
#endif
if (unlikely(!io_msg_alloc_async(req)))
return -ENOMEM;
if (req->opcode != IORING_OP_SENDMSG)
@ -715,31 +704,20 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,
iomsg->msg.msg_name = &iomsg->addr;
iomsg->msg.msg_iter.nr_segs = 0;
#ifdef CONFIG_COMPAT
if (unlikely(req->ctx->compat)) {
if (io_is_compat(req->ctx)) {
struct compat_msghdr cmsg;
ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_DEST);
if (unlikely(ret))
return ret;
ret = __get_compat_msghdr(&iomsg->msg, &cmsg, &iomsg->uaddr);
if (unlikely(ret))
return ret;
return io_recvmsg_mshot_prep(req, iomsg, cmsg.msg_namelen,
cmsg.msg_controllen);
ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_DEST,
&iomsg->uaddr);
memset(&msg, 0, sizeof(msg));
msg.msg_namelen = cmsg.msg_namelen;
msg.msg_controllen = cmsg.msg_controllen;
} else {
ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr);
}
#endif
ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST);
if (unlikely(ret))
return ret;
ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr);
if (unlikely(ret))
return ret;
return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
msg.msg_controllen);
}
@ -773,10 +751,7 @@ static int io_recvmsg_prep_setup(struct io_kiocb *req)
return 0;
}
ret = io_recvmsg_copy_hdr(req, kmsg);
if (!ret)
req->flags |= REQ_F_NEED_CLEANUP;
return ret;
return io_recvmsg_copy_hdr(req, kmsg);
}
#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
@ -787,6 +762,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
sr->done_io = 0;
sr->retry = false;
if (unlikely(sqe->file_index || sqe->addr2))
return -EINVAL;
@ -827,14 +803,16 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EINVAL;
}
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
if (io_is_compat(req->ctx))
sr->msg_flags |= MSG_CMSG_COMPAT;
#endif
sr->nr_multishot_loops = 0;
return io_recvmsg_prep_setup(req);
}
/* bits to clear in old and inherit in new cflags on bundle retry */
#define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE)
/*
* Finishes io_recv and io_recvmsg.
*
@ -854,9 +832,19 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
if (sr->flags & IORING_RECVSEND_BUNDLE) {
cflags |= io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret),
issue_flags);
if (sr->retry)
cflags = req->cqe.flags | (cflags & CQE_F_MASK);
/* bundle with no more immediate buffers, we're done */
if (req->flags & REQ_F_BL_EMPTY)
goto finish;
/* if more is available, retry and append to this one */
if (!sr->retry && kmsg->msg.msg_inq > 0 && *ret > 0) {
req->cqe.flags = cflags & ~CQE_F_MASK;
sr->len = kmsg->msg.msg_inq;
sr->done_io += *ret;
sr->retry = true;
return false;
}
} else {
cflags |= io_put_kbuf(req, *ret, issue_flags);
}
@ -1235,6 +1223,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
struct io_kiocb *notif;
zc->done_io = 0;
zc->retry = false;
req->flags |= REQ_F_POLL_NO_LAZY;
if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
@ -1273,14 +1262,13 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
zc->len = READ_ONCE(sqe->len);
zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
zc->buf_index = READ_ONCE(sqe->buf_index);
req->buf_index = READ_ONCE(sqe->buf_index);
if (zc->msg_flags & MSG_DONTWAIT)
req->flags |= REQ_F_NOWAIT;
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
if (io_is_compat(req->ctx))
zc->msg_flags |= MSG_CMSG_COMPAT;
#endif
if (unlikely(!io_msg_alloc_async(req)))
return -ENOMEM;
if (req->opcode != IORING_OP_SENDMSG_ZC)
@ -1345,24 +1333,10 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
int ret;
if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
ret = -EFAULT;
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
if (node) {
io_req_assign_buf_node(sr->notif, node);
ret = 0;
}
io_ring_submit_unlock(ctx, issue_flags);
if (unlikely(ret))
return ret;
ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter,
node->buf, (u64)(uintptr_t)sr->buf,
sr->len);
sr->notif->buf_index = req->buf_index;
ret = io_import_reg_buf(sr->notif, &kmsg->msg.msg_iter,
(u64)(uintptr_t)sr->buf, sr->len,
ITER_SOURCE, issue_flags);
if (unlikely(ret))
return ret;
kmsg->msg.sg_from_iter = io_sg_from_iter;
@ -1599,7 +1573,6 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags)
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
req_set_fail(req);
} else if (!fixed) {
fd_install(fd, file);
ret = fd;
@ -1612,14 +1585,8 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags)
if (!arg.is_empty)
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
io_req_set_res(req, ret, cflags);
return IOU_OK;
}
if (ret < 0)
return ret;
if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) &&
io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
goto retry;
if (issue_flags & IO_URING_F_MULTISHOT)
@ -1628,6 +1595,10 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags)
}
io_req_set_res(req, ret, cflags);
if (ret < 0)
req_set_fail(req);
if (!(issue_flags & IO_URING_F_MULTISHOT))
return IOU_OK;
return IOU_STOP_MULTISHOT;
}

18
io_uring/nop.c
View File

@ -16,7 +16,6 @@ struct io_nop {
struct file *file;
int result;
int fd;
int buffer;
unsigned int flags;
};
@ -40,9 +39,7 @@ int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
else
nop->fd = -1;
if (nop->flags & IORING_NOP_FIXED_BUFFER)
nop->buffer = READ_ONCE(sqe->buf_index);
else
nop->buffer = -1;
req->buf_index = READ_ONCE(sqe->buf_index);
return 0;
}
@ -64,17 +61,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
}
}
if (nop->flags & IORING_NOP_FIXED_BUFFER) {
struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
ret = -EFAULT;
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
if (node) {
io_req_assign_buf_node(req, node);
ret = 0;
}
io_ring_submit_unlock(ctx, issue_flags);
if (!io_find_buf_node(req, issue_flags))
ret = -EFAULT;
}
done:
if (ret < 0)

4
io_uring/notif.c
View File

@ -11,7 +11,7 @@
static const struct ubuf_info_ops io_ubuf_ops;
static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
{
struct io_notif_data *nd = io_notif_to_data(notif);
@ -29,7 +29,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
}
nd = nd->next;
io_req_task_complete(notif, ts);
io_req_task_complete(notif, tw);
} while (nd);
}

4
io_uring/opdef.c
View File

@ -104,7 +104,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll_queue = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_read_fixed,
.issue = io_read,
.issue = io_read_fixed,
},
[IORING_OP_WRITE_FIXED] = {
.needs_file = 1,
@ -118,7 +118,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll_queue = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_write_fixed,
.issue = io_write,
.issue = io_write_fixed,
},
[IORING_OP_POLL_ADD] = {
.needs_file = 1,

12
io_uring/opdef.h
View File

@ -7,6 +7,12 @@ struct io_issue_def {
unsigned needs_file : 1;
/* should block plug */
unsigned plug : 1;
/* supports ioprio */
unsigned ioprio : 1;
/* supports iopoll */
unsigned iopoll : 1;
/* op supports buffer selection */
unsigned buffer_select : 1;
/* hash wq insertion if file is a regular file */
unsigned hash_reg_file : 1;
/* unbound wq insertion if file is a non-regular file */
@ -15,14 +21,8 @@ struct io_issue_def {
unsigned pollin : 1;
unsigned pollout : 1;
unsigned poll_exclusive : 1;
/* op supports buffer selection */
unsigned buffer_select : 1;
/* skip auditing */
unsigned audit_skip : 1;
/* supports ioprio */
unsigned ioprio : 1;
/* supports iopoll */
unsigned iopoll : 1;
/* have to be put into the iopoll list */
unsigned iopoll_queue : 1;
/* vectored opcode, set if 1) vectored, and 2) handler needs to know */

18
io_uring/poll.c
View File

@ -220,7 +220,7 @@ static inline void io_poll_execute(struct io_kiocb *req, int res)
* req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
* poll and that the result is stored in req->cqe.
*/
static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
{
int v;
@ -288,7 +288,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
return IOU_POLL_REMOVE_POLL_USE_RES;
}
} else {
int ret = io_poll_issue(req, ts);
int ret = io_poll_issue(req, tw);
if (ret == IOU_STOP_MULTISHOT)
return IOU_POLL_REMOVE_POLL_USE_RES;
else if (ret == IOU_REQUEUE)
@ -311,11 +311,11 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
return IOU_POLL_NO_ACTION;
}
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
{
int ret;
ret = io_poll_check_events(req, ts);
ret = io_poll_check_events(req, tw);
if (ret == IOU_POLL_NO_ACTION) {
io_kbuf_recycle(req, 0);
return;
@ -335,7 +335,7 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
poll = io_kiocb_to_cmd(req, struct io_poll);
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
} else if (ret == IOU_POLL_REISSUE) {
io_req_task_submit(req, ts);
io_req_task_submit(req, tw);
return;
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
req->cqe.res = ret;
@ -343,14 +343,14 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
}
io_req_set_res(req, req->cqe.res, 0);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
} else {
io_tw_lock(req->ctx, ts);
io_tw_lock(req->ctx, tw);
if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
io_req_task_submit(req, ts);
io_req_task_submit(req, tw);
else
io_req_defer_failed(req, ret);
}

4
io_uring/poll.h
View File

@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/io_uring_types.h>
#define IO_POLL_ALLOC_CACHE_MAX 32
enum {
@ -43,4 +45,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
bool cancel_all);
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts);
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);

246
io_uring/rsrc.c
View File

@ -9,6 +9,7 @@
#include <linux/hugetlb.h>
#include <linux/compat.h>
#include <linux/io_uring.h>
#include <linux/io_uring/cmd.h>
#include <uapi/linux/io_uring.h>
@ -32,6 +33,8 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
#define IORING_MAX_FIXED_FILES (1U << 20)
#define IORING_MAX_REG_BUFFERS (1U << 14)
#define IO_CACHED_BVECS_SEGS 32
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
unsigned long page_limit, cur_pages, new_pages;
@ -101,36 +104,79 @@ static int io_buffer_validate(struct iovec *iov)
return 0;
}
static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
static void io_release_ubuf(void *priv)
{
struct io_mapped_ubuf *imu = priv;
unsigned int i;
if (node->buf) {
struct io_mapped_ubuf *imu = node->buf;
if (!refcount_dec_and_test(&imu->refs))
return;
for (i = 0; i < imu->nr_bvecs; i++)
unpin_user_page(imu->bvec[i].bv_page);
if (imu->acct_pages)
io_unaccount_mem(ctx, imu->acct_pages);
kvfree(imu);
}
for (i = 0; i < imu->nr_bvecs; i++)
unpin_user_page(imu->bvec[i].bv_page);
}
struct io_rsrc_node *io_rsrc_node_alloc(int type)
static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx,
int nr_bvecs)
{
if (nr_bvecs <= IO_CACHED_BVECS_SEGS)
return io_cache_alloc(&ctx->imu_cache, GFP_KERNEL);
return kvmalloc(struct_size_t(struct io_mapped_ubuf, bvec, nr_bvecs),
GFP_KERNEL);
}
static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
{
if (imu->nr_bvecs <= IO_CACHED_BVECS_SEGS)
io_cache_free(&ctx->imu_cache, imu);
else
kvfree(imu);
}
static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
{
if (!refcount_dec_and_test(&imu->refs))
return;
if (imu->acct_pages)
io_unaccount_mem(ctx, imu->acct_pages);
imu->release(imu->priv);
io_free_imu(ctx, imu);
}
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type)
{
struct io_rsrc_node *node;
node = kzalloc(sizeof(*node), GFP_KERNEL);
node = io_cache_alloc(&ctx->node_cache, GFP_KERNEL);
if (node) {
node->type = type;
node->refs = 1;
node->tag = 0;
node->file_ptr = 0;
}
return node;
}
__cold void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data)
bool io_rsrc_cache_init(struct io_ring_ctx *ctx)
{
const int imu_cache_size = struct_size_t(struct io_mapped_ubuf, bvec,
IO_CACHED_BVECS_SEGS);
const int node_size = sizeof(struct io_rsrc_node);
bool ret;
ret = io_alloc_cache_init(&ctx->node_cache, IO_ALLOC_CACHE_MAX,
node_size, 0);
ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX,
imu_cache_size, 0);
return ret;
}
void io_rsrc_cache_free(struct io_ring_ctx *ctx)
{
io_alloc_cache_free(&ctx->node_cache, kfree);
io_alloc_cache_free(&ctx->imu_cache, kfree);
}
__cold void io_rsrc_data_free(struct io_ring_ctx *ctx,
struct io_rsrc_data *data)
{
if (!data->nr)
return;
@ -203,7 +249,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
err = -EBADF;
break;
}
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
if (!node) {
err = -ENOMEM;
fput(file);
@ -449,19 +495,17 @@ void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
switch (node->type) {
case IORING_RSRC_FILE:
if (io_slot_file(node))
fput(io_slot_file(node));
fput(io_slot_file(node));
break;
case IORING_RSRC_BUFFER:
if (node->buf)
io_buffer_unmap(ctx, node);
io_buffer_unmap(ctx, node->buf);
break;
default:
WARN_ON_ONCE(1);
break;
}
kfree(node);
io_cache_free(&ctx->node_cache, node);
}
int io_sqe_files_unregister(struct io_ring_ctx *ctx)
@ -523,7 +567,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
goto fail;
}
ret = -ENOMEM;
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
if (!node) {
fput(file);
goto fail;
@ -728,10 +772,9 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
if (!iov->iov_base)
return NULL;
node = io_rsrc_node_alloc(IORING_RSRC_BUFFER);
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!node)
return ERR_PTR(-ENOMEM);
node->buf = NULL;
ret = -ENOMEM;
pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
@ -748,10 +791,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
coalesced = io_coalesce_buffer(&pages, &nr_pages, &data);
}
imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
imu = io_alloc_imu(ctx, nr_pages);
if (!imu)
goto done;
imu->nr_bvecs = nr_pages;
ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
if (ret) {
unpin_user_pages(pages, nr_pages);
@ -762,8 +806,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
/* store original address for later verification */
imu->ubuf = (unsigned long) iov->iov_base;
imu->len = iov->iov_len;
imu->nr_bvecs = nr_pages;
imu->folio_shift = PAGE_SHIFT;
imu->release = io_release_ubuf;
imu->priv = imu;
imu->is_kbuf = false;
imu->dir = IO_IMU_DEST | IO_IMU_SOURCE;
if (coalesced)
imu->folio_shift = data.folio_shift;
refcount_set(&imu->refs, 1);
@ -781,9 +828,9 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
}
done:
if (ret) {
kvfree(imu);
if (node)
io_put_rsrc_node(ctx, node);
if (imu)
io_free_imu(ctx, imu);
io_cache_free(&ctx->node_cache, node);
node = ERR_PTR(ret);
}
kvfree(pages);
@ -860,7 +907,102 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
return ret;
}
int io_import_fixed(int ddir, struct iov_iter *iter,
int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
void (*release)(void *), unsigned int index,
unsigned int issue_flags)
{
struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx;
struct io_rsrc_data *data = &ctx->buf_table;
struct req_iterator rq_iter;
struct io_mapped_ubuf *imu;
struct io_rsrc_node *node;
struct bio_vec bv, *bvec;
u16 nr_bvecs;
int ret = 0;
io_ring_submit_lock(ctx, issue_flags);
if (index >= data->nr) {
ret = -EINVAL;
goto unlock;
}
index = array_index_nospec(index, data->nr);
if (data->nodes[index]) {
ret = -EBUSY;
goto unlock;
}
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!node) {
ret = -ENOMEM;
goto unlock;
}
nr_bvecs = blk_rq_nr_phys_segments(rq);
imu = io_alloc_imu(ctx, nr_bvecs);
if (!imu) {
kfree(node);
ret = -ENOMEM;
goto unlock;
}
imu->ubuf = 0;
imu->len = blk_rq_bytes(rq);
imu->acct_pages = 0;
imu->folio_shift = PAGE_SHIFT;
imu->nr_bvecs = nr_bvecs;
refcount_set(&imu->refs, 1);
imu->release = release;
imu->priv = rq;
imu->is_kbuf = true;
imu->dir = 1 << rq_data_dir(rq);
bvec = imu->bvec;
rq_for_each_bvec(bv, rq, rq_iter)
*bvec++ = bv;
node->buf = imu;
data->nodes[index] = node;
unlock:
io_ring_submit_unlock(ctx, issue_flags);
return ret;
}
EXPORT_SYMBOL_GPL(io_buffer_register_bvec);
int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
unsigned int issue_flags)
{
struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx;
struct io_rsrc_data *data = &ctx->buf_table;
struct io_rsrc_node *node;
int ret = 0;
io_ring_submit_lock(ctx, issue_flags);
if (index >= data->nr) {
ret = -EINVAL;
goto unlock;
}
index = array_index_nospec(index, data->nr);
node = data->nodes[index];
if (!node) {
ret = -EINVAL;
goto unlock;
}
if (!node->buf->is_kbuf) {
ret = -EBUSY;
goto unlock;
}
io_put_rsrc_node(ctx, node);
data->nodes[index] = NULL;
unlock:
io_ring_submit_unlock(ctx, issue_flags);
return ret;
}
EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec);
static int io_import_fixed(int ddir, struct iov_iter *iter,
struct io_mapped_ubuf *imu,
u64 buf_addr, size_t len)
{
@ -874,6 +1016,8 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
/* not inside the mapped region */
if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
return -EFAULT;
if (!(imu->dir & (1 << ddir)))
return -EFAULT;
/*
* Might not be a start of buffer, set size appropriately
@ -886,8 +1030,8 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
/*
* Don't use iov_iter_advance() here, as it's really slow for
* using the latter parts of a big fixed buffer - it iterates
* over each segment manually. We can cheat a bit here, because
* we know that:
* over each segment manually. We can cheat a bit here for user
* registered nodes, because we know that:
*
* 1) it's a BVEC iter, we set it up
* 2) all bvecs are the same in size, except potentially the
@ -901,8 +1045,15 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
*/
const struct bio_vec *bvec = imu->bvec;
/*
* Kernel buffer bvecs, on the other hand, don't necessarily
* have the size property of user registered ones, so we have
* to use the slow iter advance.
*/
if (offset < bvec->bv_len) {
iter->iov_offset = offset;
} else if (imu->is_kbuf) {
iov_iter_advance(iter, offset);
} else {
unsigned long seg_skip;
@ -919,6 +1070,35 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
return 0;
}
inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
unsigned issue_flags)
{
struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
if (req->flags & REQ_F_BUF_NODE)
return req->buf_node;
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
if (node)
io_req_assign_buf_node(req, node);
io_ring_submit_unlock(ctx, issue_flags);
return node;
}
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags)
{
struct io_rsrc_node *node;
node = io_find_buf_node(req, issue_flags);
if (!node)
return -EFAULT;
return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
}
/* Lock two rings at once. The rings must be different! */
static void lock_two_rings(struct io_ring_ctx *ctx1, struct io_ring_ctx *ctx2)
{
@ -1002,7 +1182,7 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
if (!src_node) {
dst_node = NULL;
} else {
dst_node = io_rsrc_node_alloc(IORING_RSRC_BUFFER);
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!dst_node) {
ret = -ENOMEM;
goto out_free;

24
io_uring/rsrc.h
View File

@ -2,6 +2,7 @@
#ifndef IOU_RSRC_H
#define IOU_RSRC_H
#include <linux/io_uring_types.h>
#include <linux/lockdep.h>
enum {
@ -20,6 +21,11 @@ struct io_rsrc_node {
};
};
enum {
IO_IMU_DEST = 1 << ITER_DEST,
IO_IMU_SOURCE = 1 << ITER_SOURCE,
};
struct io_mapped_ubuf {
u64 ubuf;
unsigned int len;
@ -27,6 +33,10 @@ struct io_mapped_ubuf {
unsigned int folio_shift;
refcount_t refs;
unsigned long acct_pages;
void (*release)(void *);
void *priv;
bool is_kbuf;
u8 dir;
struct bio_vec bvec[] __counted_by(nr_bvecs);
};
@ -39,14 +49,18 @@ struct io_imu_folio_data {
unsigned int nr_folios;
};
struct io_rsrc_node *io_rsrc_node_alloc(int type);
bool io_rsrc_cache_init(struct io_ring_ctx *ctx);
void io_rsrc_cache_free(struct io_ring_ctx *ctx);
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type);
void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
int io_import_fixed(int ddir, struct iov_iter *iter,
struct io_mapped_ubuf *imu,
u64 buf_addr, size_t len);
struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
unsigned issue_flags);
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags);
int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg);
int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
@ -77,7 +91,7 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data
static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
{
lockdep_assert_held(&ctx->uring_lock);
if (node && !--node->refs)
if (!--node->refs)
io_free_rsrc_node(ctx, node);
}

200
io_uring/rw.c
View File

@ -49,24 +49,16 @@ static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask)
return false;
}
#ifdef CONFIG_COMPAT
static int io_iov_compat_buffer_select_prep(struct io_rw *rw)
{
struct compat_iovec __user *uiov;
compat_ssize_t clen;
struct compat_iovec __user *uiov = u64_to_user_ptr(rw->addr);
struct compat_iovec iov;
uiov = u64_to_user_ptr(rw->addr);
if (!access_ok(uiov, sizeof(*uiov)))
if (copy_from_user(&iov, uiov, sizeof(iov)))
return -EFAULT;
if (__get_user(clen, &uiov->iov_len))
return -EFAULT;
if (clen < 0)
return -EINVAL;
rw->len = clen;
rw->len = iov.iov_len;
return 0;
}
#endif
static int io_iov_buffer_select_prep(struct io_kiocb *req)
{
@ -77,10 +69,8 @@ static int io_iov_buffer_select_prep(struct io_kiocb *req)
if (rw->len != 1)
return -EINVAL;
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
if (io_is_compat(req->ctx))
return io_iov_compat_buffer_select_prep(rw);
#endif
uiov = u64_to_user_ptr(rw->addr);
if (copy_from_user(&iov, uiov, sizeof(*uiov)))
@ -89,41 +79,24 @@ static int io_iov_buffer_select_prep(struct io_kiocb *req)
return 0;
}
static int __io_import_iovec(int ddir, struct io_kiocb *req,
struct io_async_rw *io,
unsigned int issue_flags)
static int io_import_vec(int ddir, struct io_kiocb *req,
struct io_async_rw *io,
const struct iovec __user *uvec,
size_t uvec_segs)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
int ret, nr_segs;
struct iovec *iov;
void __user *buf;
int nr_segs, ret;
size_t sqe_len;
buf = u64_to_user_ptr(rw->addr);
sqe_len = rw->len;
if (!def->vectored || req->flags & REQ_F_BUFFER_SELECT) {
if (io_do_buffer_select(req)) {
buf = io_buffer_select(req, &sqe_len, issue_flags);
if (!buf)
return -ENOBUFS;
rw->addr = (unsigned long) buf;
rw->len = sqe_len;
}
return import_ubuf(ddir, buf, sqe_len, &io->iter);
}
if (io->free_iovec) {
nr_segs = io->free_iov_nr;
iov = io->free_iovec;
} else {
iov = &io->fast_iov;
nr_segs = 1;
iov = &io->fast_iov;
}
ret = __import_iovec(ddir, buf, sqe_len, nr_segs, &iov, &io->iter,
req->ctx->compat);
ret = __import_iovec(ddir, uvec, uvec_segs, nr_segs, &iov, &io->iter,
io_is_compat(req->ctx));
if (unlikely(ret < 0))
return ret;
if (iov) {
@ -135,13 +108,35 @@ static int __io_import_iovec(int ddir, struct io_kiocb *req,
return 0;
}
static inline int io_import_iovec(int rw, struct io_kiocb *req,
struct io_async_rw *io,
unsigned int issue_flags)
static int __io_import_rw_buffer(int ddir, struct io_kiocb *req,
struct io_async_rw *io,
unsigned int issue_flags)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
void __user *buf = u64_to_user_ptr(rw->addr);
size_t sqe_len = rw->len;
if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT))
return io_import_vec(ddir, req, io, buf, sqe_len);
if (io_do_buffer_select(req)) {
buf = io_buffer_select(req, &sqe_len, issue_flags);
if (!buf)
return -ENOBUFS;
rw->addr = (unsigned long) buf;
rw->len = sqe_len;
}
return import_ubuf(ddir, buf, sqe_len, &io->iter);
}
static inline int io_import_rw_buffer(int rw, struct io_kiocb *req,
struct io_async_rw *io,
unsigned int issue_flags)
{
int ret;
ret = __io_import_iovec(rw, req, io, issue_flags);
ret = __io_import_rw_buffer(rw, req, io, issue_flags);
if (unlikely(ret < 0))
return ret;
@ -212,20 +207,6 @@ static int io_rw_alloc_async(struct io_kiocb *req)
return 0;
}
static int io_prep_rw_setup(struct io_kiocb *req, int ddir, bool do_import)
{
struct io_async_rw *rw;
if (io_rw_alloc_async(req))
return -ENOMEM;
if (!do_import || io_do_buffer_select(req))
return 0;
rw = req->async_data;
return io_import_iovec(ddir, req, rw, 0);
}
static inline void io_meta_save_state(struct io_async_rw *io)
{
io->meta_state.seed = io->meta.seed;
@ -267,14 +248,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
return ret;
}
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
int ddir, bool do_import)
static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
int ddir)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
unsigned ioprio;
u64 attr_type_mask;
int ret;
if (io_rw_alloc_async(req))
return -ENOMEM;
rw->kiocb.ki_pos = READ_ONCE(sqe->off);
/* used for fixed read/write too - just read unconditionally */
req->buf_index = READ_ONCE(sqe->buf_index);
@ -300,10 +284,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
rw->addr = READ_ONCE(sqe->addr);
rw->len = READ_ONCE(sqe->len);
rw->flags = READ_ONCE(sqe->rw_flags);
ret = io_prep_rw_setup(req, ddir, do_import);
if (unlikely(ret))
return ret;
attr_type_mask = READ_ONCE(sqe->attr_type_mask);
if (attr_type_mask) {
@ -314,31 +294,50 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return -EINVAL;
attr_ptr = READ_ONCE(sqe->attr_ptr);
ret = io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask);
return io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask);
}
return ret;
return 0;
}
static int io_rw_do_import(struct io_kiocb *req, int ddir)
{
if (io_do_buffer_select(req))
return 0;
return io_import_rw_buffer(ddir, req, req->async_data, 0);
}
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
int ddir)
{
int ret;
ret = __io_prep_rw(req, sqe, ddir);
if (unlikely(ret))
return ret;
return io_rw_do_import(req, ddir);
}
int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
return io_prep_rw(req, sqe, ITER_DEST, true);
return io_prep_rw(req, sqe, ITER_DEST);
}
int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
return io_prep_rw(req, sqe, ITER_SOURCE, true);
return io_prep_rw(req, sqe, ITER_SOURCE);
}
static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe,
int ddir)
{
const bool do_import = !(req->flags & REQ_F_BUFFER_SELECT);
int ret;
ret = io_prep_rw(req, sqe, ddir, do_import);
ret = io_prep_rw(req, sqe, ddir);
if (unlikely(ret))
return ret;
if (do_import)
if (!(req->flags & REQ_F_BUFFER_SELECT))
return 0;
/*
@ -358,38 +357,30 @@ int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return io_prep_rwv(req, sqe, ITER_SOURCE);
}
static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe,
static int io_init_rw_fixed(struct io_kiocb *req, unsigned int issue_flags,
int ddir)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
struct io_async_rw *io;
struct io_async_rw *io = req->async_data;
int ret;
ret = io_prep_rw(req, sqe, ddir, false);
if (unlikely(ret))
return ret;
if (io->bytes_done)
return 0;
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
if (!node)
return -EFAULT;
io_req_assign_buf_node(req, node);
io = req->async_data;
ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len);
ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir,
issue_flags);
iov_iter_save_state(&io->iter, &io->iter_state);
return ret;
}
int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
return io_prep_rw_fixed(req, sqe, ITER_DEST);
return __io_prep_rw(req, sqe, ITER_DEST);
}
int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
return io_prep_rw_fixed(req, sqe, ITER_SOURCE);
return __io_prep_rw(req, sqe, ITER_SOURCE);
}
/*
@ -405,7 +396,7 @@ int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (!(req->flags & REQ_F_BUFFER_SELECT))
return -EINVAL;
ret = io_prep_rw(req, sqe, ITER_DEST, false);
ret = __io_prep_rw(req, sqe, ITER_DEST);
if (unlikely(ret))
return ret;
@ -519,7 +510,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
return res;
}
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
struct kiocb *kiocb = &rw->kiocb;
@ -536,7 +527,7 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0);
io_req_rw_cleanup(req, 0);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
}
static void io_complete_rw(struct kiocb *kiocb, long res)
@ -638,6 +629,7 @@ static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
*/
static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)
{
struct io_kiocb *req = cmd_to_io_kiocb(rw);
struct kiocb *kiocb = &rw->kiocb;
struct file *file = kiocb->ki_filp;
ssize_t ret = 0;
@ -653,6 +645,8 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)
if ((kiocb->ki_flags & IOCB_NOWAIT) &&
!(kiocb->ki_filp->f_flags & O_NONBLOCK))
return -EAGAIN;
if ((req->flags & REQ_F_BUF_NODE) && req->buf_node->buf->is_kbuf)
return -EFAULT;
ppos = io_kiocb_ppos(kiocb);
@ -864,7 +858,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
loff_t *ppos;
if (io_do_buffer_select(req)) {
ret = io_import_iovec(ITER_DEST, req, io, issue_flags);
ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
if (unlikely(ret < 0))
return ret;
}
@ -1155,6 +1149,28 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)
}
}
int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags)
{
int ret;
ret = io_init_rw_fixed(req, issue_flags, ITER_DEST);
if (unlikely(ret))
return ret;
return io_read(req, issue_flags);
}
int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags)
{
int ret;
ret = io_init_rw_fixed(req, issue_flags, ITER_SOURCE);
if (unlikely(ret))
return ret;
return io_write(req, issue_flags);
}
void io_rw_fail(struct io_kiocb *req)
{
int res;

5
io_uring/rw.h
View File

@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/io_uring_types.h>
#include <linux/pagemap.h>
struct io_meta_state {
@ -37,9 +38,11 @@ int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_read(struct io_kiocb *req, unsigned int issue_flags);
int io_write(struct io_kiocb *req, unsigned int issue_flags);
int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags);
int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags);
void io_readv_writev_cleanup(struct io_kiocb *req);
void io_rw_fail(struct io_kiocb *req);
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
void io_rw_cache_free(const void *entry);

3
io_uring/splice.c
View File

@ -51,7 +51,8 @@ void io_splice_cleanup(struct io_kiocb *req)
{
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
io_put_rsrc_node(req->ctx, sp->rsrc_node);
if (sp->rsrc_node)
io_put_rsrc_node(req->ctx, sp->rsrc_node);
}
static struct file *io_splice_get_file(struct io_kiocb *req,

16
io_uring/timeout.c
View File

@ -65,7 +65,7 @@ static inline bool io_timeout_finish(struct io_timeout *timeout,
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_timeout_data *data = req->async_data;
@ -82,7 +82,7 @@ static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
}
}
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
}
static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
@ -154,9 +154,9 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
io_flush_killed_timeouts(&list, 0);
}
static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
{
io_tw_lock(link->ctx, ts);
io_tw_lock(link->ctx, tw);
while (link) {
struct io_kiocb *nxt = link->link;
long res = -ECANCELED;
@ -165,7 +165,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
res = link->cqe.res;
link->link = NULL;
io_req_set_res(link, res, 0);
io_req_task_complete(link, ts);
io_req_task_complete(link, tw);
link = nxt;
}
}
@ -312,7 +312,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
return 0;
}
static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_kiocb *prev = timeout->prev;
@ -330,11 +330,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t
ret = -ECANCELED;
}
io_req_set_res(req, ret ?: -ETIME, 0);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
io_put_req(prev);
} else {
io_req_set_res(req, -ETIME, 0);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
}
}

31
io_uring/uring_cmd.c
View File

@ -102,7 +102,7 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
}
EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)
static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
unsigned int flags = IO_URING_F_COMPLETE_DEFER;
@ -199,21 +199,9 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (ioucmd->flags & ~IORING_URING_CMD_MASK)
return -EINVAL;
if (ioucmd->flags & IORING_URING_CMD_FIXED) {
struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
u16 index = READ_ONCE(sqe->buf_index);
if (ioucmd->flags & IORING_URING_CMD_FIXED)
req->buf_index = READ_ONCE(sqe->buf_index);
node = io_rsrc_node_lookup(&ctx->buf_table, index);
if (unlikely(!node))
return -EFAULT;
/*
* Pi node upfront, prior to io_uring_cmd_import_fixed()
* being called. This prevents destruction of the mapped buffer
* we'll need at actual import time.
*/
io_req_assign_buf_node(req, node);
}
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
return io_uring_cmd_prep_setup(req, sqe);
@ -237,7 +225,7 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
issue_flags |= IO_URING_F_SQE128;
if (ctx->flags & IORING_SETUP_CQE32)
issue_flags |= IO_URING_F_CQE32;
if (ctx->compat)
if (io_is_compat(ctx))
issue_flags |= IO_URING_F_COMPAT;
if (ctx->flags & IORING_SETUP_IOPOLL) {
if (!file->f_op->uring_cmd_iopoll)
@ -257,16 +245,13 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
}
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd)
struct iov_iter *iter,
struct io_uring_cmd *ioucmd,
unsigned int issue_flags)
{
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
struct io_rsrc_node *node = req->buf_node;
/* Must have had rsrc_node assigned at prep time */
if (node)
return io_import_fixed(rw, iter, node->buf, ubuf, len);
return -EFAULT;
return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags);
}
EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);

56
io_uring/waitid.c
View File

@ -16,7 +16,7 @@
#include "waitid.h"
#include "../kernel/exit.h"
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts);
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw);
#define IO_WAITID_CANCEL_FLAG BIT(31)
#define IO_WAITID_REF_MASK GENMASK(30, 0)
@ -42,7 +42,6 @@ static void io_waitid_free(struct io_kiocb *req)
req->flags &= ~REQ_F_ASYNC_DATA;
}
#ifdef CONFIG_COMPAT
static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo)
{
struct compat_siginfo __user *infop;
@ -67,7 +66,6 @@ static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo)
ret = false;
goto done;
}
#endif
static bool io_waitid_copy_si(struct io_kiocb *req, int signo)
{
@ -77,10 +75,8 @@ static bool io_waitid_copy_si(struct io_kiocb *req, int signo)
if (!iw->infop)
return true;
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
if (io_is_compat(req->ctx))
return io_waitid_compat_copy_si(iw, signo);
#endif
if (!user_write_access_begin(iw->infop, sizeof(*iw->infop)))
return false;
@ -132,7 +128,7 @@ static void io_waitid_complete(struct io_kiocb *req, int ret)
io_req_set_res(req, ret, 0);
}
static bool __io_waitid_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
static bool __io_waitid_cancel(struct io_kiocb *req)
{
struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
struct io_waitid_async *iwa = req->async_data;
@ -158,49 +154,13 @@ static bool __io_waitid_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
unsigned int issue_flags)
{
struct hlist_node *tmp;
struct io_kiocb *req;
int nr = 0;
if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED))
return -ENOENT;
io_ring_submit_lock(ctx, issue_flags);
hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
if (req->cqe.user_data != cd->data &&
!(cd->flags & IORING_ASYNC_CANCEL_ANY))
continue;
if (__io_waitid_cancel(ctx, req))
nr++;
if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
break;
}
io_ring_submit_unlock(ctx, issue_flags);
if (nr)
return nr;
return -ENOENT;
return io_cancel_remove(ctx, cd, issue_flags, &ctx->waitid_list, __io_waitid_cancel);
}
bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
bool cancel_all)
{
struct hlist_node *tmp;
struct io_kiocb *req;
bool found = false;
lockdep_assert_held(&ctx->uring_lock);
hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
if (!io_match_task_safe(req, tctx, cancel_all))
continue;
hlist_del_init(&req->hash_node);
__io_waitid_cancel(ctx, req);
found = true;
}
return found;
return io_cancel_remove_all(ctx, tctx, &ctx->waitid_list, cancel_all, __io_waitid_cancel);
}
static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
@ -221,13 +181,13 @@ static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
return true;
}
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_waitid_async *iwa = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
int ret;
io_tw_lock(ctx, ts);
io_tw_lock(ctx, tw);
ret = __do_wait(&iwa->wo);
@ -257,7 +217,7 @@ static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
}
io_waitid_complete(req, ret);
io_req_task_complete(req, ts);
io_req_task_complete(req, tw);
}
static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,

1
tools/testing/selftests/Makefile
View File

@ -113,6 +113,7 @@ endif
TARGETS += tmpfs
TARGETS += tpm2
TARGETS += tty
TARGETS += ublk
TARGETS += uevent
TARGETS += user_events
TARGETS += vDSO

3
tools/testing/selftests/ublk/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
kublk
/tools
*-verify.state

16
tools/testing/selftests/ublk/Makefile Normal file
View File

@ -0,0 +1,16 @@
# SPDX-License-Identifier: GPL-2.0
CFLAGS += -O3 -Wl,-no-as-needed -Wall -I $(top_srcdir)
LDLIBS += -lpthread -lm -luring
TEST_PROGS := test_null_01.sh
TEST_PROGS += test_loop_01.sh
TEST_PROGS += test_loop_02.sh
TEST_PROGS += test_loop_03.sh
TEST_PROGS += test_loop_04.sh
TEST_GEN_PROGS_EXTENDED = kublk
include ../lib.mk
$(TEST_GEN_PROGS_EXTENDED): kublk.c null.c file_backed.c

1
tools/testing/selftests/ublk/config Normal file
View File

@ -0,0 +1 @@
CONFIG_BLK_DEV_UBLK=m

220
tools/testing/selftests/ublk/file_backed.c Normal file
View File

@ -0,0 +1,220 @@
// SPDX-License-Identifier: GPL-2.0
#include "kublk.h"
static void backing_file_tgt_deinit(struct ublk_dev *dev)
{
int i;
for (i = 1; i < dev->nr_fds; i++) {
fsync(dev->fds[i]);
close(dev->fds[i]);
}
}
static int backing_file_tgt_init(struct ublk_dev *dev)
{
int fd, i;
assert(dev->nr_fds == 1);
for (i = 0; i < dev->tgt.nr_backing_files; i++) {
char *file = dev->tgt.backing_file[i];
unsigned long bytes;
struct stat st;
ublk_dbg(UBLK_DBG_DEV, "%s: file %d: %s\n", __func__, i, file);
fd = open(file, O_RDWR | O_DIRECT);
if (fd < 0) {
ublk_err("%s: backing file %s can't be opened: %s\n",
__func__, file, strerror(errno));
return -EBADF;
}
if (fstat(fd, &st) < 0) {
close(fd);
return -EBADF;
}
if (S_ISREG(st.st_mode))
bytes = st.st_size;
else if (S_ISBLK(st.st_mode)) {
if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
return -1;
} else {
return -EINVAL;
}
dev->tgt.backing_file_size[i] = bytes;
dev->fds[dev->nr_fds] = fd;
dev->nr_fds += 1;
}
return 0;
}
static enum io_uring_op ublk_to_uring_op(const struct ublksrv_io_desc *iod, int zc)
{
unsigned ublk_op = ublksrv_get_op(iod);
if (ublk_op == UBLK_IO_OP_READ)
return zc ? IORING_OP_READ_FIXED : IORING_OP_READ;
else if (ublk_op == UBLK_IO_OP_WRITE)
return zc ? IORING_OP_WRITE_FIXED : IORING_OP_WRITE;
assert(0);
}
static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
{
int zc = ublk_queue_use_zc(q);
enum io_uring_op op = ublk_to_uring_op(iod, zc);
struct io_uring_sqe *reg;
struct io_uring_sqe *rw;
struct io_uring_sqe *ureg;
if (!zc) {
rw = ublk_queue_alloc_sqe(q);
if (!rw)
return -ENOMEM;
io_uring_prep_rw(op, rw, 1 /*fds[1]*/,
(void *)iod->addr,
iod->nr_sectors << 9,
iod->start_sector << 9);
io_uring_sqe_set_flags(rw, IOSQE_FIXED_FILE);
q->io_inflight++;
/* bit63 marks us as tgt io */
rw->user_data = build_user_data(tag, op, UBLK_IO_TGT_NORMAL, 1);
return 0;
}
ublk_queue_alloc_sqe3(q, &reg, &rw, &ureg);
io_uring_prep_buf_register(reg, 0, tag, q->q_id, tag);
reg->user_data = build_user_data(tag, 0xfe, 1, 1);
reg->flags |= IOSQE_CQE_SKIP_SUCCESS;
reg->flags |= IOSQE_IO_LINK;
io_uring_prep_rw(op, rw, 1 /*fds[1]*/, 0,
iod->nr_sectors << 9,
iod->start_sector << 9);
rw->buf_index = tag;
rw->flags |= IOSQE_FIXED_FILE;
rw->flags |= IOSQE_IO_LINK;
rw->user_data = build_user_data(tag, op, UBLK_IO_TGT_ZC_OP, 1);
q->io_inflight++;
io_uring_prep_buf_unregister(ureg, 0, tag, q->q_id, tag);
ureg->user_data = build_user_data(tag, 0xff, UBLK_IO_TGT_ZC_BUF, 1);
q->io_inflight++;
return 0;
}
static int loop_queue_tgt_io(struct ublk_queue *q, int tag)
{
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
unsigned ublk_op = ublksrv_get_op(iod);
struct io_uring_sqe *sqe;
switch (ublk_op) {
case UBLK_IO_OP_FLUSH:
sqe = ublk_queue_alloc_sqe(q);
if (!sqe)
return -ENOMEM;
io_uring_prep_sync_file_range(sqe, 1 /*fds[1]*/,
iod->nr_sectors << 9,
iod->start_sector << 9,
IORING_FSYNC_DATASYNC);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE);
q->io_inflight++;
sqe->user_data = build_user_data(tag, ublk_op, UBLK_IO_TGT_NORMAL, 1);
break;
case UBLK_IO_OP_WRITE_ZEROES:
case UBLK_IO_OP_DISCARD:
return -ENOTSUP;
case UBLK_IO_OP_READ:
case UBLK_IO_OP_WRITE:
loop_queue_tgt_rw_io(q, iod, tag);
break;
default:
return -EINVAL;
}
ublk_dbg(UBLK_DBG_IO, "%s: tag %d ublk io %x %llx %u\n", __func__, tag,
iod->op_flags, iod->start_sector, iod->nr_sectors << 9);
return 1;
}
static int ublk_loop_queue_io(struct ublk_queue *q, int tag)
{
int queued = loop_queue_tgt_io(q, tag);
if (queued < 0)
ublk_complete_io(q, tag, queued);
return 0;
}
static void ublk_loop_io_done(struct ublk_queue *q, int tag,
const struct io_uring_cqe *cqe)
{
int cqe_tag = user_data_to_tag(cqe->user_data);
unsigned tgt_data = user_data_to_tgt_data(cqe->user_data);
int res = cqe->res;
if (res < 0 || tgt_data == UBLK_IO_TGT_NORMAL)
goto complete;
if (tgt_data == UBLK_IO_TGT_ZC_OP) {
ublk_set_io_res(q, tag, cqe->res);
goto exit;
}
assert(tgt_data == UBLK_IO_TGT_ZC_BUF);
res = ublk_get_io_res(q, tag);
complete:
assert(tag == cqe_tag);
ublk_complete_io(q, tag, res);
exit:
q->io_inflight--;
}
static int ublk_loop_tgt_init(struct ublk_dev *dev)
{
unsigned long long bytes;
int ret;
struct ublk_params p = {
.types = UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DMA_ALIGN,
.basic = {
.logical_bs_shift = 9,
.physical_bs_shift = 12,
.io_opt_shift = 12,
.io_min_shift = 9,
.max_sectors = dev->dev_info.max_io_buf_bytes >> 9,
},
.dma = {
.alignment = 511,
},
};
assert(dev->tgt.nr_backing_files == 1);
ret = backing_file_tgt_init(dev);
if (ret)
return ret;
bytes = dev->tgt.backing_file_size[0];
dev->tgt.dev_size = bytes;
p.basic.dev_sectors = bytes >> 9;
dev->tgt.params = p;
return 0;
}
const struct ublk_tgt_ops loop_tgt_ops = {
.name = "loop",
.init_tgt = ublk_loop_tgt_init,
.deinit_tgt = backing_file_tgt_deinit,
.queue_io = ublk_loop_queue_io,
.tgt_io_done = ublk_loop_io_done,
};

1110
tools/testing/selftests/ublk/kublk.c Normal file
View File

File diff suppressed because it is too large Load Diff

326
tools/testing/selftests/ublk/kublk.h Normal file
View File

@ -0,0 +1,326 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef KUBLK_INTERNAL_H
#define KUBLK_INTERNAL_H
#include <unistd.h>
#include <stdlib.h>
#include <assert.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <pthread.h>
#include <getopt.h>
#include <limits.h>
#include <poll.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/inotify.h>
#include <sys/wait.h>
#include <sys/eventfd.h>
#include <liburing.h>
#include <linux/ublk_cmd.h>
#define __maybe_unused __attribute__((unused))
#define MAX_BACK_FILES 4
#ifndef min
#define min(a, b) ((a) < (b) ? (a) : (b))
#endif
/****************** part 1: libublk ********************/
#define CTRL_DEV "/dev/ublk-control"
#define UBLKC_DEV "/dev/ublkc"
#define UBLKB_DEV "/dev/ublkb"
#define UBLK_CTRL_RING_DEPTH 32
#define ERROR_EVTFD_DEVID -2
/* queue idle timeout */
#define UBLKSRV_IO_IDLE_SECS 20
#define UBLK_IO_MAX_BYTES 65536
#define UBLK_MAX_QUEUES 4
#define UBLK_QUEUE_DEPTH 128
#define UBLK_IO_TGT_NORMAL 0
#define UBLK_IO_TGT_ZC_BUF 1
#define UBLK_IO_TGT_ZC_OP 2
#define UBLK_DBG_DEV (1U << 0)
#define UBLK_DBG_QUEUE (1U << 1)
#define UBLK_DBG_IO_CMD (1U << 2)
#define UBLK_DBG_IO (1U << 3)
#define UBLK_DBG_CTRL_CMD (1U << 4)
#define UBLK_LOG (1U << 5)
struct ublk_dev;
struct ublk_queue;
struct dev_ctx {
char tgt_type[16];
unsigned long flags;
unsigned nr_hw_queues;
unsigned queue_depth;
int dev_id;
int nr_files;
char *files[MAX_BACK_FILES];
unsigned int logging:1;
unsigned int all:1;
int _evtfd;
};
struct ublk_ctrl_cmd_data {
__u32 cmd_op;
#define CTRL_CMD_HAS_DATA 1
#define CTRL_CMD_HAS_BUF 2
__u32 flags;
__u64 data[2];
__u64 addr;
__u32 len;
};
struct ublk_io {
char *buf_addr;
#define UBLKSRV_NEED_FETCH_RQ (1UL << 0)
#define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1)
#define UBLKSRV_IO_FREE (1UL << 2)
unsigned short flags;
unsigned short refs; /* used by target code only */
int result;
};
struct ublk_tgt_ops {
const char *name;
int (*init_tgt)(struct ublk_dev *);
void (*deinit_tgt)(struct ublk_dev *);
int (*queue_io)(struct ublk_queue *, int tag);
void (*tgt_io_done)(struct ublk_queue *,
int tag, const struct io_uring_cqe *);
};
struct ublk_tgt {
unsigned long dev_size;
unsigned int sq_depth;
unsigned int cq_depth;
const struct ublk_tgt_ops *ops;
struct ublk_params params;
int nr_backing_files;
unsigned long backing_file_size[MAX_BACK_FILES];
char backing_file[MAX_BACK_FILES][PATH_MAX];
};
struct ublk_queue {
int q_id;
int q_depth;
unsigned int cmd_inflight;
unsigned int io_inflight;
struct ublk_dev *dev;
const struct ublk_tgt_ops *tgt_ops;
char *io_cmd_buf;
struct io_uring ring;
struct ublk_io ios[UBLK_QUEUE_DEPTH];
#define UBLKSRV_QUEUE_STOPPING (1U << 0)
#define UBLKSRV_QUEUE_IDLE (1U << 1)
#define UBLKSRV_NO_BUF (1U << 2)
#define UBLKSRV_ZC (1U << 3)
unsigned state;
pid_t tid;
pthread_t thread;
};
struct ublk_dev {
struct ublk_tgt tgt;
struct ublksrv_ctrl_dev_info dev_info;
struct ublk_queue q[UBLK_MAX_QUEUES];
int fds[MAX_BACK_FILES + 1]; /* fds[0] points to /dev/ublkcN */
int nr_fds;
int ctrl_fd;
struct io_uring ring;
};
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t)&((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
unsigned long __mptr = (unsigned long)(ptr); \
((type *)(__mptr - offsetof(type, member))); })
#endif
#define round_up(val, rnd) \
(((val) + ((rnd) - 1)) & ~((rnd) - 1))
extern unsigned int ublk_dbg_mask;
extern int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag);
static inline int is_target_io(__u64 user_data)
{
return (user_data & (1ULL << 63)) != 0;
}
static inline __u64 build_user_data(unsigned tag, unsigned op,
unsigned tgt_data, unsigned is_target_io)
{
assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16));
return tag | (op << 16) | (tgt_data << 24) | (__u64)is_target_io << 63;
}
static inline unsigned int user_data_to_tag(__u64 user_data)
{
return user_data & 0xffff;
}
static inline unsigned int user_data_to_op(__u64 user_data)
{
return (user_data >> 16) & 0xff;
}
static inline unsigned int user_data_to_tgt_data(__u64 user_data)
{
return (user_data >> 24) & 0xffff;
}
static inline void ublk_err(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
}
static inline void ublk_log(const char *fmt, ...)
{
if (ublk_dbg_mask & UBLK_LOG) {
va_list ap;
va_start(ap, fmt);
vfprintf(stdout, fmt, ap);
}
}
static inline void ublk_dbg(int level, const char *fmt, ...)
{
if (level & ublk_dbg_mask) {
va_list ap;
va_start(ap, fmt);
vfprintf(stdout, fmt, ap);
}
}
static inline struct io_uring_sqe *ublk_queue_alloc_sqe(struct ublk_queue *q)
{
unsigned left = io_uring_sq_space_left(&q->ring);
if (left < 1)
io_uring_submit(&q->ring);
return io_uring_get_sqe(&q->ring);
}
static inline void ublk_queue_alloc_sqe3(struct ublk_queue *q,
struct io_uring_sqe **sqe1, struct io_uring_sqe **sqe2,
struct io_uring_sqe **sqe3)
{
struct io_uring *r = &q->ring;
unsigned left = io_uring_sq_space_left(r);
if (left < 3)
io_uring_submit(r);
*sqe1 = io_uring_get_sqe(r);
*sqe2 = io_uring_get_sqe(r);
*sqe3 = io_uring_get_sqe(r);
}
static inline void io_uring_prep_buf_register(struct io_uring_sqe *sqe,
int dev_fd, int tag, int q_id, __u64 index)
{
struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd;
io_uring_prep_read(sqe, dev_fd, 0, 0, 0);
sqe->opcode = IORING_OP_URING_CMD;
sqe->flags |= IOSQE_FIXED_FILE;
sqe->cmd_op = UBLK_U_IO_REGISTER_IO_BUF;
cmd->tag = tag;
cmd->addr = index;
cmd->q_id = q_id;
}
static inline void io_uring_prep_buf_unregister(struct io_uring_sqe *sqe,
int dev_fd, int tag, int q_id, __u64 index)
{
struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd;
io_uring_prep_read(sqe, dev_fd, 0, 0, 0);
sqe->opcode = IORING_OP_URING_CMD;
sqe->flags |= IOSQE_FIXED_FILE;
sqe->cmd_op = UBLK_U_IO_UNREGISTER_IO_BUF;
cmd->tag = tag;
cmd->addr = index;
cmd->q_id = q_id;
}
static inline void *ublk_get_sqe_cmd(const struct io_uring_sqe *sqe)
{
return (void *)&sqe->cmd;
}
static inline void ublk_set_io_res(struct ublk_queue *q, int tag, int res)
{
q->ios[tag].result = res;
}
static inline int ublk_get_io_res(const struct ublk_queue *q, unsigned tag)
{
return q->ios[tag].result;
}
static inline void ublk_mark_io_done(struct ublk_io *io, int res)
{
io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE);
io->result = res;
}
static inline const struct ublksrv_io_desc *ublk_get_iod(const struct ublk_queue *q, int tag)
{
return (struct ublksrv_io_desc *)&(q->io_cmd_buf[tag * sizeof(struct ublksrv_io_desc)]);
}
static inline void ublk_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op)
{
__u32 *addr = (__u32 *)&sqe->off;
addr[0] = cmd_op;
addr[1] = 0;
}
static inline int ublk_complete_io(struct ublk_queue *q, unsigned tag, int res)
{
struct ublk_io *io = &q->ios[tag];
ublk_mark_io_done(io, res);
return ublk_queue_io_cmd(q, io, tag);
}
static inline int ublk_queue_use_zc(const struct ublk_queue *q)
{
return q->state & UBLKSRV_ZC;
}
extern const struct ublk_tgt_ops null_tgt_ops;
extern const struct ublk_tgt_ops loop_tgt_ops;
#endif

38
tools/testing/selftests/ublk/null.c Normal file
View File

@ -0,0 +1,38 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include "kublk.h"
static int ublk_null_tgt_init(struct ublk_dev *dev)
{
const struct ublksrv_ctrl_dev_info *info = &dev->dev_info;
unsigned long dev_size = 250UL << 30;
dev->tgt.dev_size = dev_size;
dev->tgt.params = (struct ublk_params) {
.types = UBLK_PARAM_TYPE_BASIC,
.basic = {
.logical_bs_shift = 9,
.physical_bs_shift = 12,
.io_opt_shift = 12,
.io_min_shift = 9,
.max_sectors = info->max_io_buf_bytes >> 9,
.dev_sectors = dev_size >> 9,
},
};
return 0;
}
static int ublk_null_queue_io(struct ublk_queue *q, int tag)
{
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
ublk_complete_io(q, tag, iod->nr_sectors << 9);
return 0;
}
const struct ublk_tgt_ops null_tgt_ops = {
.name = "null",
.init_tgt = ublk_null_tgt_init,
.queue_io = ublk_null_queue_io,
};

113
tools/testing/selftests/ublk/test_common.sh Executable file
View File

@ -0,0 +1,113 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
_create_backfile() {
local my_size=$1
local my_file=`mktemp ublk_bpf_${my_size}_XXXXX`
truncate -s ${my_size} ${my_file}
echo $my_file
}
_remove_backfile() {
local file=$1
[ -f "$file" ] && rm -f $file
}
_create_tmp_dir() {
local my_file=`mktemp -d ublk_bpf_dir_XXXXX`
echo $my_file
}
_remove_tmp_dir() {
local dir=$1
[ -d "$dir" ] && rmdir $dir
}
_mkfs_mount_test()
{
local dev=$1
local err_code=0
local mnt_dir=`_create_tmp_dir`
mkfs.ext4 -F $dev > /dev/null 2>&1
err_code=$?
if [ $err_code -ne 0 ]; then
return $err_code
fi
mount -t ext4 $dev $mnt_dir > /dev/null 2>&1
umount $dev
err_code=$?
_remove_tmp_dir $mnt_dir
if [ $err_code -ne 0 ]; then
return $err_code
fi
}
_check_root() {
local ksft_skip=4
if [ $UID != 0 ]; then
echo please run this as root >&2
exit $ksft_skip
fi
}
_remove_ublk_devices() {
${UBLK_PROG} del -a
}
_get_ublk_dev_state() {
${UBLK_PROG} list -n "$1" | grep "state" | awk '{print $11}'
}
_get_ublk_daemon_pid() {
${UBLK_PROG} list -n "$1" | grep "pid" | awk '{print $7}'
}
_prep_test() {
_check_root
local type=$1
shift 1
echo "ublk $type: $@"
}
_show_result()
{
if [ $2 -ne 0 ]; then
echo "$1 : [FAIL]"
else
echo "$1 : [PASS]"
fi
}
_cleanup_test() {
${UBLK_PROG} del -n $1
}
_add_ublk_dev() {
local kublk_temp=`mktemp /tmp/kublk-XXXXXX`
${UBLK_PROG} add $@ > ${kublk_temp} 2>&1
if [ $? -ne 0 ]; then
echo "fail to add ublk dev $@"
exit -1
fi
local dev_id=`grep "dev id" ${kublk_temp} | awk -F '[ :]' '{print $3}'`
udevadm settle
rm -f ${kublk_temp}
echo ${dev_id}
}
_have_feature()
{
if $UBLK_PROG "features" | grep $1 > /dev/null 2>&1; then
return 0
fi
return 1
}
export UBLK_PROG=$(pwd)/kublk

31
tools/testing/selftests/ublk/test_loop_01.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. test_common.sh
TID="loop_01"
ERR_CODE=0
_prep_test "loop" "write and verify test"
backfile_0=`_create_backfile 256M`
dev_id=`_add_ublk_dev -t loop $backfile_0`
# run fio over the ublk disk
fio --name=write_and_verify \
--filename=/dev/ublkb${dev_id} \
--ioengine=libaio --iodepth=16 \
--rw=write \
--size=256M \
--direct=1 \
--verify=crc32c \
--do_verify=1 \
--bs=4k > /dev/null 2>&1
ERR_CODE=$?
_cleanup_test ${dev_id} "loop"
_remove_backfile $backfile_0
_show_result $TID $ERR_CODE

22
tools/testing/selftests/ublk/test_loop_02.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. test_common.sh
TID="loop_02"
ERR_CODE=0
_prep_test "loop" "mkfs & mount & umount"
backfile_0=`_create_backfile 256M`
dev_id=`_add_ublk_dev -t loop $backfile_0`
_mkfs_mount_test /dev/ublkb${dev_id}
ERR_CODE=$?
_cleanup_test ${dev_id} "loop"
_remove_backfile $backfile_0
_show_result $TID $ERR_CODE

33
tools/testing/selftests/ublk/test_loop_03.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. test_common.sh
TID="loop_03"
ERR_CODE=0
_have_feature "ZERO_COPY" || exit 4
_prep_test "loop" "write and verify over zero copy"
backfile_0=`_create_backfile 256M`
dev_id=`_add_ublk_dev -t loop $backfile_0 -z`
# run fio over the ublk disk
fio --name=write_and_verify \
--filename=/dev/ublkb${dev_id} \
--ioengine=libaio --iodepth=64 \
--rw=write \
--size=256M \
--direct=1 \
--verify=crc32c \
--do_verify=1 \
--bs=4k > /dev/null 2>&1
ERR_CODE=$?
_cleanup_test ${dev_id} "loop"
_remove_backfile $backfile_0
_show_result $TID $ERR_CODE

22
tools/testing/selftests/ublk/test_loop_04.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. test_common.sh
TID="loop_04"
ERR_CODE=0
_prep_test "loop" "mkfs & mount & umount with zero copy"
backfile_0=`_create_backfile 256M`
dev_id=`_add_ublk_dev -t loop -z $backfile_0`
_mkfs_mount_test /dev/ublkb${dev_id}
ERR_CODE=$?
_cleanup_test ${dev_id} "loop"
_remove_backfile $backfile_0
_show_result $TID $ERR_CODE

19
tools/testing/selftests/ublk/test_null_01.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. test_common.sh
TID="null_01"
ERR_CODE=0
_prep_test "null" "basic IO test"
dev_id=`_add_ublk_dev -t null`
# run fio over the two disks
fio --name=job1 --filename=/dev/ublkb${dev_id} --ioengine=libaio --rw=readwrite --iodepth=32 --size=256M > /dev/null 2>&1
ERR_CODE=$?
_cleanup_test ${dev_id} "null"
_show_result $TID $ERR_CODE