github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 13:14:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

ublk: move auto buffer register handling into one dedicated helper

Browse Source 
Move check & clearing UBLK_IO_FLAG_AUTO_BUF_REG to
ublk_handle_auto_buf_reg(), also return buffer index from this helper.

Also move ublk_set_auto_buf_reg() to this single helper too.

Add ublk_config_io_buf() for setting up ublk io buffer, covers both
ublk buffer copy or auto buffer register.

Reviewed-by: Caleb Sander Mateos <csander@purestorage.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250713143415.2857561-7-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei 2025-07-13 22:34:01 +08:00 committed by Jens Axboe
parent 7ebdba87cf
commit 52460dda3a
1 changed files with 71 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
drivers/block/ublk_drv.c
View File

@ -48,6 +48,8 @@
#define UBLK_MINORS (1U << MINORBITS) #define UBLK_MINORS (1U << MINORBITS)
#define UBLK_INVALID_BUF_IDX ((u16)-1)
/* private ioctl command mirror */ /* private ioctl command mirror */
#define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC) #define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)
#define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE)
@ -2009,10 +2011,47 @@ static inline int ublk_check_cmd_op(u32 cmd_op)
return 0; return 0;
} }
static inline int ublk_set_auto_buf_reg(struct io_uring_cmd *cmd)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
pdu->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr));
if (pdu->buf.reserved0 || pdu->buf.reserved1)
return -EINVAL;
if (pdu->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK)
return -EINVAL;
return 0;
}
static int ublk_handle_auto_buf_reg(struct ublk_io *io,
struct io_uring_cmd *cmd,
u16 *buf_idx)
{
if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) {
io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG;
/*
* `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ`
* and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same
* `io_ring_ctx`.
*
* If this uring_cmd's io_ring_ctx isn't same with the
* one for registering the buffer, it is ublk server's
* responsibility for unregistering the buffer, otherwise
* this ublk request gets stuck.
*/
if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd))
*buf_idx = io->buf_index;
}
return ublk_set_auto_buf_reg(cmd);
}
/* Once we return, `io->req` can't be used any more */ /* Once we return, `io->req` can't be used any more */
static inline struct request * static inline struct request *
ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd, ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd)
unsigned long buf_addr)
{ {
struct request *req = io->req; struct request *req = io->req;
@ -2020,11 +2059,22 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd,
io->flags |= UBLK_IO_FLAG_ACTIVE; io->flags |= UBLK_IO_FLAG_ACTIVE;
/* now this cmd slot is owned by ublk driver */ /* now this cmd slot is owned by ublk driver */
io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV; io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
io->addr = buf_addr;
return req; return req;
} }
static inline int
ublk_config_io_buf(const struct ublk_queue *ubq, struct ublk_io *io,
struct io_uring_cmd *cmd, unsigned long buf_addr,
u16 *buf_idx)
{
if (ublk_support_auto_buf_reg(ubq))
return ublk_handle_auto_buf_reg(io, cmd, buf_idx);
io->addr = buf_addr;
return 0;
}
static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
unsigned int issue_flags, unsigned int issue_flags,
struct ublk_queue *ubq, unsigned int tag) struct ublk_queue *ubq, unsigned int tag)
@ -2040,20 +2090,6 @@ static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
io_uring_cmd_mark_cancelable(cmd, issue_flags); io_uring_cmd_mark_cancelable(cmd, issue_flags);
} }
static inline int ublk_set_auto_buf_reg(struct io_uring_cmd *cmd)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
pdu->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr));
if (pdu->buf.reserved0 || pdu->buf.reserved1)
return -EINVAL;
if (pdu->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK)
return -EINVAL;
return 0;
}
static void ublk_io_release(void *priv) static void ublk_io_release(void *priv)
{ {
struct request *rq = priv; struct request *rq = priv;
@ -2174,13 +2210,11 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq,
goto out; goto out;
} }
if (ublk_support_auto_buf_reg(ubq)) { ublk_fill_io_cmd(io, cmd);
ret = ublk_set_auto_buf_reg(cmd); ret = ublk_config_io_buf(ubq, io, cmd, buf_addr, NULL);
if (ret) if (ret)
goto out; goto out;
}
ublk_fill_io_cmd(io, cmd, buf_addr);
WRITE_ONCE(io->task, get_task_struct(current)); WRITE_ONCE(io->task, get_task_struct(current));
ublk_mark_io_ready(ub, ubq); ublk_mark_io_ready(ub, ubq);
out: out:
@ -2212,35 +2246,13 @@ static int ublk_check_commit_and_fetch(const struct ublk_queue *ubq,
return 0; return 0;
} }
static int ublk_commit_and_fetch(const struct ublk_queue *ubq, static void ublk_commit_and_fetch(const struct ublk_queue *ubq,
struct ublk_io *io, struct io_uring_cmd *cmd, struct ublk_io *io, struct io_uring_cmd *cmd,
struct request *req, unsigned int issue_flags, struct request *req, unsigned int issue_flags,
__u64 zone_append_lba) __u64 zone_append_lba, u16 buf_idx)
{ {
if (ublk_support_auto_buf_reg(ubq)) { if (buf_idx != UBLK_INVALID_BUF_IDX)
int ret; io_buffer_unregister_bvec(cmd, buf_idx, issue_flags);
/*
* `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ`
* and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same
* `io_ring_ctx`.
*
* If this uring_cmd's io_ring_ctx isn't same with the
* one for registering the buffer, it is ublk server's
* responsibility for unregistering the buffer, otherwise
* this ublk request gets stuck.
*/
if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) {
if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd))
io_buffer_unregister_bvec(cmd, io->buf_index,
issue_flags);
io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG;
}
ret = ublk_set_auto_buf_reg(cmd);
if (ret)
return ret;
}
if (req_op(req) == REQ_OP_ZONE_APPEND) if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = zone_append_lba; req->__sector = zone_append_lba;
@ -2249,7 +2261,6 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq,
ublk_sub_req_ref(io, req); ublk_sub_req_ref(io, req);
else else
__ublk_complete_rq(req); __ublk_complete_rq(req);
return 0;
} }
static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io, static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io,
@ -2274,6 +2285,7 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
unsigned int issue_flags, unsigned int issue_flags,
const struct ublksrv_io_cmd *ub_cmd) const struct ublksrv_io_cmd *ub_cmd)
{ {
u16 buf_idx = UBLK_INVALID_BUF_IDX;
struct ublk_device *ub = cmd->file->private_data; struct ublk_device *ub = cmd->file->private_data;
struct ublk_queue *ubq; struct ublk_queue *ubq;
struct ublk_io *io; struct ublk_io *io;
@ -2353,9 +2365,10 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
if (ret) if (ret)
goto out; goto out;
io->res = ub_cmd->result; io->res = ub_cmd->result;
req = ublk_fill_io_cmd(io, cmd, ub_cmd->addr); req = ublk_fill_io_cmd(io, cmd);
ret = ublk_commit_and_fetch(ubq, io, cmd, req, issue_flags, ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, &buf_idx);
ub_cmd->zone_append_lba); ublk_commit_and_fetch(ubq, io, cmd, req, issue_flags,
ub_cmd->zone_append_lba, buf_idx);
if (ret) if (ret)
goto out; goto out;
break; break;
@ -2365,7 +2378,9 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
* uring_cmd active first and prepare for handling new requeued * uring_cmd active first and prepare for handling new requeued
* request * request
*/ */
req = ublk_fill_io_cmd(io, cmd, ub_cmd->addr); req = ublk_fill_io_cmd(io, cmd);
ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, NULL);
WARN_ON_ONCE(ret);
if (likely(ublk_get_data(ubq, io, req))) { if (likely(ublk_get_data(ubq, io, req))) {
__ublk_prep_compl_io_cmd(io, req); __ublk_prep_compl_io_cmd(io, req);
return UBLK_IO_RES_OK; return UBLK_IO_RES_OK;