github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

for-7.4/io_uring-dmabuf-20261023

Browse Source 
-----BEGIN PGP SIGNATURE-----
 
 iQJEBAABCAAuFiEEwPw5LcreJtl1+l5K99NY+ylx4KYFAmrbkFsQHGF4Ym9lQGtl
 cm5lbC5kawAKCRD301j7KXHgpiHpD/9MdThoGJTZ3mKgaZPd7lOgkxWL51CY/ULB
 X9zzyfHP9hpXxwXqpSGm8XhN7cmSDXGaCYvhVPS2YQQv0458s1UGMWDi8PJNulFS
 6M8ng6Hex2z/Uq+u6vMkruz+hH76c3ajo6kGOhJnWye+/Rq1bqqFcYBWMpLBE/yj
 lp8JGG92HyMnsIo5gzBHbUoNEc+X0+jz88mk0JRZoONJD4cNb1S1bW5v+uAk0Nvc
 /qPE9AXU2DnngvbzhfKGza/4Lkx9WKBXFmxJSiG9ZH6GJn3yndBNBi7RAODku+/E
 i+SiukHPDhn3M5cWZuJ8ilTpVWRCnnsbhavM1PN2tvcW6IfZrAz+m321+JVkzKu0
 cLUDvmz2WS0+PcodXGn3KcGVYt7vAjtQVZwZLbq2012u5xSvZN/VaeiVDIRr1uC/
 hQPiD51ZpsTt/2lAyrFGmLBsskNWc0Xhd6ePX7euNCdhfHj5UuWKcXUiQeeGiUvT
 X2JyQAJSY9VZ6eO02w77bta0II+3SSnZlA6xVmE3+Nwv2z6qr3b31gcfMIiap3Tu
 jUkKvDuB9Dw9hhgLr1t6BCuaPS3ktyd1F3dBvbs8YXyyeworU2KGtSF7mraJxts8
 GCH+DOAR1j29qMi4/bjhxOKQYAKzhWbptCiJgyReL6V+uFO6JxAeWn3qn36mjezp
 BuHNTX1A5g==
 =hlkK
 -----END PGP SIGNATURE-----

Merge tag 'for-7.4/io_uring-dmabuf-20261023' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring dma-buf support from Jens Axboe:
 "This adds native dma-buf support to the I/O path, exposed through
  io_uring. It's been in development for a while and should be ready to
  go now.

   - dma-buf file I/O infrastructure: a dma_buf_io_ctx created through a
     new ->init_dma_buf_io_ctx file operation, and a dma_buf_io_map
     holding the device DMA mapping. The two are split so a mapping can
     be invalidated and re-established underneath a live context, with
     the map counting in-flight requests so it can be torn down safely.

   - A new iov_iter type for dmabuf maps, and a dma map backed bio type
     for the block layer, with raw block device support on top.

   - nvme-pci backend implementing dma-buf backed requests, including
     SGL support.

   - io_uring dmabuf backed registered buffers, built on a reworked
     buffer registration structure with an extended update path, regbuf
     import flags, an uncloneable flag for buffer types that can't be
     cloned, and cloning support for registered dma-bufs"

* tag 'for-7.4/io_uring-dmabuf-20261023' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
  io_uring/dma-buf: don't fail fixed buffers without dma-buf support
  nvme-pci: detach dma-bufs only on removal
  block: reject dma-bufs larger than 4GiB
  io_uring/dma-buf: allow cloning registered dma-bufs
  dma-buf-io: document dma_buf_io_init_map
  dma-buf-io: fix typos in kernel docs
  block: fix indention for dma-buf checks
  io_uring/rsrc: add dmabuf backed registered buffers
  io_uring/rsrc: add regbuf import flags
  io_uring/rsrc: add uncloneable regbuf flag
  io_uring/rsrc: extend buffer update
  io_uring/rsrc: introduce buf registration structure
  nvme-pci: add SGL support for the dmabuf path
  nvme-pci: implement dma-buf backed requests
  nvme-pci: rename nvme_pci_sgl_set_data to nvme_pci_dma_iter_set_sgl
  block: add dma-buf support for raw bdev
  block: introduce dma map backed bio type
  block: always adjust bi_offset on bio_advance_iter
  iov_iter: add iterator type for dmabuf maps
  dma-buf: introduce initial file I/O infrastructure
This commit is contained in:
Linus Torvalds 2026-10-27 07:46:05 -10:00
commit ed02f4f7ad
31 changed files with 1414 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
block/bio.c
View File

@ -881,7 +881,11 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
bio->bi_write_stream = bio_src->bi_write_stream;
bio->bi_bvec_gap_bit = bio_src->bi_bvec_gap_bit;
bio->bi_iter = bio_src->bi_iter;
bio->bi_io_vec = bio_src->bi_io_vec;
if (op_is_dmabuf(bio->bi_opf))
bio->bi_dmabuf_map = bio_src->bi_dmabuf_map;
else
bio->bi_io_vec = bio_src->bi_io_vec;
if (bio->bi_bdev) {
if (bio->bi_bdev == bio_src->bi_bdev &&
@ -1204,16 +1208,23 @@ EXPORT_SYMBOL_GPL(__bio_release_pages);
bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter)
{
if (!iov_iter_is_bvec(iter))
if (!iov_iter_is_bvec(iter) && !iov_iter_is_dmabuf_map(iter))
return false;
WARN_ON_ONCE(bio->bi_max_vecs);
static_assert(offsetof(struct bio, bi_io_vec) ==
offsetof(struct bio, bi_dmabuf_map));
static_assert(offsetof(struct iov_iter, bvec) ==
offsetof(struct iov_iter, dmabuf_map));
bio->bi_io_vec = (struct bio_vec *)iter->bvec;
bio->bi_iter.bi_idx = 0;
bio->bi_iter.bi_offset = iter->iov_offset;
bio->bi_iter.bi_size = iov_iter_count(iter);
bio_set_flag(bio, BIO_CLONED);
if (iov_iter_is_dmabuf_map(iter))
bio->bi_opf |= REQ_NOMERGE | REQ_DMABUF;
return true;
}

52
block/blk-merge.c
View File

@ -9,6 +9,7 @@
#include <linux/blk-integrity.h>
#include <linux/part_stat.h>
#include <linux/blk-cgroup.h>
#include <linux/dma-buf-io.h>
#include <trace/events/block.h>
@ -319,6 +320,41 @@ static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv,
return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len);
}
static inline int bio_split_io_at_dmabuf(struct bio *bio,
const struct queue_limits *lim, unsigned *segs,
unsigned max_bytes, unsigned len_align_mask,
unsigned start_align_mask)
{
unsigned bytes = min(bio->bi_iter.bi_size, max_bytes);
unsigned seg_shift = bio->bi_dmabuf_map->min_seg_shift;
unsigned offset = bio->bi_iter.bi_offset & ((1U << seg_shift) - 1);
u64 max_segs_bytes;
/*
* dma-buf maps don't expose the underlying segmentation, but they're
* guaranteed to not have alignment gaps, we only need to check the
* start and length alignment.
*/
if ((bio->bi_iter.bi_offset & start_align_mask) ||
(bio->bi_iter.bi_size & len_align_mask))
return -EINVAL;
/* Presented as a single contiguous range into the dma-buf */
*segs = 1;
/*
* Limit by the number of segments by using the minimal segment size.
* Any I/O consisting of N full segments should be able to cover at
* least N multiplied by the segment size. It's stricter than walking
* the segments and might cause extra splitting.
*/
max_segs_bytes = (u64)lim->max_segments << seg_shift;
bytes = min_t(u64, bytes, max_segs_bytes - offset);
if (bytes != bio->bi_iter.bi_size)
return bytes;
return 0;
}
/**
* bio_split_io_at - check if and where to split a bio
* @bio: [in] bio to be split
@ -346,6 +382,19 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
len_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1);
}
if (op_is_dmabuf(bio->bi_opf)) {
int ret;
ret = bio_split_io_at_dmabuf(bio, lim, &nsegs, max_bytes,
len_align_mask, start_align_mask);
if (ret < 0)
return ret;
if (!ret)
goto out;
bytes = ret;
goto split;
}
bio_for_each_bvec(bv, bio, iter) {
if (bv.bv_offset & start_align_mask ||
bv.bv_len & len_align_mask)
@ -376,6 +425,7 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
bvprvp = &bvprv;
}
out:
*segs = nsegs;
bio->bi_bvec_gap_bit = ffs(gaps);
return 0;
@ -493,6 +543,8 @@ unsigned int blk_recalc_rq_segments(struct request *rq)
if (!rq->bio)
return 0;
if (op_is_dmabuf(rq->bio->bi_opf))
return 1;
switch (bio_op(rq->bio)) {
case REQ_OP_DISCARD:

6
block/blk-mq.c
View File

@ -3150,6 +3150,12 @@ void blk_mq_submit_bio(struct bio *bio)
goto queue_exit;
}
if (unlikely(op_is_dmabuf(bio->bi_opf) &&
bdev_has_integrity_csum(bio->bi_bdev))) {
bio_endio_status(bio, BLK_STS_NOTSUPP);
goto queue_exit;
}
if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) {
bio_endio_status(bio, BLK_STS_NOTSUPP);
goto queue_exit;

29
block/fops.c
View File

@ -19,6 +19,7 @@
#include <linux/iomap.h>
#include <linux/module.h>
#include <linux/io_uring/cmd.h>
#include <linux/dma-buf-io.h>
#include "blk.h"
static inline struct inode *bdev_file_inode(struct file *file)
@ -365,7 +366,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
* Users don't rely on the iterator being in any particular
* state for async I/O returning -EIOCBQUEUED, hence we can
* avoid expensive iov_iter_advance(). Bypass
* bio_iov_iter_get_pages() and set the bvec directly.
* bio_iov_iter_get_pages() and set the bvec/dmabuf directly.
*/
if (!bio_iov_iter_set(bio, iter)) {
ret = blkdev_iov_iter_get_pages(bio, iter, bdev);
@ -727,6 +728,9 @@ static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)
if (iocb->ki_flags & IOCB_NOWAIT)
return -EAGAIN;
if (unlikely(iov_iter_is_dmabuf_map(from)))
return -EOPNOTSUPP;
/*
* Take i_rwsem to avoid racing with set_blocksize() changing
* i_blkbits/folio order and truncating the page cache.
@ -872,6 +876,11 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
if (ret < 0 || !count)
goto reexpand;
}
if (unlikely(iov_iter_is_dmabuf_map(to))) {
if (!ret)
ret = -EOPNOTSUPP;
goto reexpand;
}
/*
* Take i_rwsem and invalidate_lock to avoid racing with set_blocksize
@ -999,6 +1008,23 @@ static int blkdev_mmap_prepare(struct vm_area_desc *desc)
return generic_file_mmap_prepare(desc);
}
static int blkdev_init_dma_buf_io_ctx(struct file *file,
struct dma_buf_io_ctx *ctx)
{
struct block_device *bdev = file_bdev(file);
struct gendisk *disk = bdev->bd_disk;
if (!(file->f_flags & O_DIRECT))
return -EINVAL;
if (ctx->dmabuf->size > UINT_MAX)
return -EINVAL;
if (!disk->fops->init_dma_buf_io_ctx)
return -EOPNOTSUPP;
if (bdev_has_integrity_csum(bdev))
return -EOPNOTSUPP;
return disk->fops->init_dma_buf_io_ctx(bdev, ctx);
}
const struct file_operations def_blk_fops = {
.open = blkdev_open,
.release = blkdev_release,
@ -1017,6 +1043,7 @@ const struct file_operations def_blk_fops = {
.fallocate = blkdev_fallocate,
.uring_cmd = blkdev_uring_cmd,
.fop_flags = FOP_BUFFER_RASYNC | FOP_DONTCACHE,
.init_dma_buf_io_ctx = blkdev_init_dma_buf_io_ctx,
};
static __init int blkdev_init(void)

2
drivers/dma-buf/Makefile
View File

@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \
dma-fence-unwrap.o dma-resv.o dma-buf-mapping.o
dma-fence-unwrap.o dma-resv.o dma-buf-mapping.o dma-buf-io.o
obj-$(CONFIG_DMABUF_HEAPS) += dma-heap.o
obj-$(CONFIG_DMABUF_HEAPS) += heaps/
obj-$(CONFIG_SYNC_FILE) += sync_file.o

241
drivers/dma-buf/dma-buf-io.c Normal file
View File

@ -0,0 +1,241 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Common infrastructure for supporing dma-buf in the I/O path.
*
* Copyright (C) 2026 Pavel Begunkov <asml.silence@gmail.com>
*/
#include <linux/dma-buf-io.h>
#include <linux/dma-resv.h>
static void dma_buf_io_map_release(struct dma_buf_io_map *map)
{
struct dma_buf_io_ctx *ctx = map->ctx;
dma_resv_assert_held(ctx->dmabuf->resv);
percpu_ref_exit(&map->refs);
ctx->dev_ops->unmap(ctx, map);
}
static void dma_buf_io_map_refs_cb(struct percpu_ref *ref)
{
struct dma_buf_io_map *map = container_of(ref, struct dma_buf_io_map, refs);
complete(&map->drained);
}
static void dma_buf_io_kill_map(struct dma_buf_io_ctx *ctx)
{
struct dma_buf_io_map *map;
dma_resv_assert_held(ctx->dmabuf->resv);
map = rcu_dereference_protected(ctx->map,
dma_resv_held(ctx->dmabuf->resv));
if (!map)
return;
rcu_assign_pointer(ctx->map, NULL);
percpu_ref_kill(&map->refs);
/* make sure the map is not visible via ctx->map */
synchronize_rcu_expedited();
wait_for_completion(&map->drained);
dma_buf_io_map_release(map);
}
/**
* dma_buf_io_init_map - Initialise the map.
* @ctx: [in] dma-buf I/O context the map is created for
* @map: [in] dma-buf I/O map
* @sgt: [in] the backing scatter-gather table
*
* Importer's helper that initialises @map for the provided I/O context
* and scatter-gather table. The importer should call it once for each map
* on creation.
*
* Return: 0 on success, or a negative errno code otherwise.
*/
int dma_buf_io_init_map(struct dma_buf_io_ctx *ctx, struct dma_buf_io_map *map,
struct sg_table *sgt)
{
unsigned seg_shift = ~0U;
struct scatterlist *sg;
unsigned long tmp;
int ret;
for_each_sgtable_dma_sg(sgt, sg, tmp)
seg_shift = min(seg_shift, __ffs(sg_dma_len(sg)));
ret = percpu_ref_init(&map->refs, dma_buf_io_map_refs_cb, 0, GFP_KERNEL);
if (ret)
return ret;
init_completion(&map->drained);
map->min_seg_shift = seg_shift;
map->ctx = ctx;
return 0;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_io_init_map, "DMA_BUF");
static struct dma_buf_io_map *__dma_buf_io_create_map(struct dma_buf_io_ctx *ctx)
{
struct dma_buf *dmabuf = ctx->dmabuf;
struct dma_buf_io_map *map;
long ret;
dma_resv_assert_held(dmabuf->resv);
if (ctx->is_dead)
return ERR_PTR(-ENOENT);
map = __dma_buf_io_get_map(ctx);
if (map)
return map;
map = ctx->dev_ops->map(ctx);
if (IS_ERR(map))
return map;
if (WARN_ON_ONCE(!map->min_seg_shift))
return ERR_PTR(-EFAULT);
ret = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL,
true, MAX_SCHEDULE_TIMEOUT);
if (ret <= 0) {
if (!ret)
ret = -EAGAIN;
dma_buf_io_map_release(map);
return ERR_PTR(ret);
}
/* get a reference for the caller */
percpu_ref_get(&map->refs);
rcu_assign_pointer(ctx->map, map);
return map;
}
/**
* dma_buf_io_create_map - Create a new or reference an existing map.
* @ctx: [in] dma-buf I/O context
*
* A slow path part of dma_buf_io_get_map() and shouldn't be used directly.
* Behaves in the same way but is allowed to block.
*
* Returns a map on success, or ERR_PTR encoded errno otherwise.
*/
struct dma_buf_io_map *dma_buf_io_create_map(struct dma_buf_io_ctx *ctx)
{
struct dma_buf_io_map *map;
int ret;
ret = dma_resv_lock_interruptible(ctx->dmabuf->resv, NULL);
if (ret)
return ERR_PTR(ret);
map = __dma_buf_io_create_map(ctx);
dma_resv_unlock(ctx->dmabuf->resv);
return map;
}
static void dma_buf_io_ctx_shutdown(struct dma_buf_io_ctx *ctx)
{
dma_resv_lock(ctx->dmabuf->resv, NULL);
ctx->is_dead = true;
dma_buf_io_kill_map(ctx);
dma_resv_unlock(ctx->dmabuf->resv);
}
/*
* dma_buf_io_ctx_create - Create a new dma-buf I/O context for the given file.
* @file: [in] file for which the context is created
* @dmabuf: [in] buffer to create a context from
* @dir: [in] direction of DMA transfers
* @out_ctx: [out] resulting ctx
*
* Create a new I/O context from the given DMA buffer for the given file.
* It only works with files implementing ->init_dma_buf_io_ctx and fails
* with -EOPNOTSUPP otherwise. Once created, the user can create a map
* and use it to issue I/O, such as read/write requests, to @file.
*/
int dma_buf_io_ctx_create(struct file *file,
struct dma_buf *dmabuf,
enum dma_data_direction dir,
struct dma_buf_io_ctx **out_ctx)
{
struct dma_buf_io_ctx *ctx;
int ret;
if (!file->f_op->init_dma_buf_io_ctx)
return -EOPNOTSUPP;
ctx = kzalloc_obj(*ctx);
if (!ctx)
return -ENOMEM;
ctx->dir = dir;
ctx->dmabuf = dmabuf;
get_dma_buf(dmabuf);
ret = file->f_op->init_dma_buf_io_ctx(file, ctx);
if (ret) {
kfree(ctx);
dma_buf_put(dmabuf);
return ret;
}
if (WARN_ON_ONCE(!ctx->dev_ops ||
!ctx->dev_ops->map ||
!ctx->dev_ops->unmap ||
!ctx->dev_ops->release))
return -EINVAL;
*out_ctx = ctx;
return 0;
}
/**
* dma_buf_io_ctx_release - Destroy the I/O context
* @ctx: [in] dma-buf I/O context
*
* Destroy the context. It makes sure all maps are waited on and destroyed,
* and can block for I/O to complete. It detaches from the importer and
* destroys all its resources via the ->release callback, and frees the
* context.
*/
void dma_buf_io_ctx_release(struct dma_buf_io_ctx *ctx)
{
dma_buf_io_ctx_shutdown(ctx);
if (WARN_ON_ONCE(rcu_dereference_protected(ctx->map, true)))
return;
ctx->dev_ops->release(ctx);
dma_buf_put(ctx->dmabuf);
kfree(ctx);
}
/**
* dma_buf_io_invalidate_mappings - Invalidate the current map
* @ctx: [in] dma-buf I/O context
*
* Importer's helper for invalidating the current map. Waits for all I/O
* using the map to complete and destroys the map. Called with the
* dma-buf reservation lock held.
*/
void dma_buf_io_invalidate_mappings(struct dma_buf_io_ctx *ctx)
{
dma_buf_io_kill_map(ctx);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_io_invalidate_mappings, "DMA_BUF");
/**
* dma_buf_io_detach - Detach the @ctx from the importer.
* @ctx: [in] dma-buf I/O context
*
* Importer's helper for removing the I/O context. It kills all maps and
* prevents creation of new ones. Upon return, it's safe for the importer
* to release all its resources such as dma-buf attachments.
*/
void dma_buf_io_detach(struct dma_buf_io_ctx *ctx)
{
dma_buf_io_ctx_shutdown(ctx);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_io_detach, "DMA_BUF");

6
drivers/md/dm-io-rewind.c
View File

@ -113,10 +113,12 @@ static inline void dm_bio_rewind_iter(const struct bio *bio,
iter->bi_sector -= bytes >> 9;
/* No advance means no rewind */
if (bio_no_advance_iter(bio))
if (bio_no_advance_iter(bio)) {
iter->bi_size += bytes;
else
iter->bi_offset -= bytes;
} else {
dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
}
}
/**

1
drivers/nvme/host/Makefile
View File

@ -21,6 +21,7 @@ nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o
nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o
nvme-y += pci.o
nvme-$(CONFIG_DMA_SHARED_BUFFER) += pci-dmabuf.o
nvme-fabrics-y += fabrics.o

12
drivers/nvme/host/core.c
View File

@ -2713,6 +2713,17 @@ static int nvme_report_zones(struct gendisk *disk, sector_t sector,
#define nvme_report_zones NULL
#endif /* CONFIG_BLK_DEV_ZONED */
static int nvme_init_dma_buf_io_ctx(struct block_device *bdev,
struct dma_buf_io_ctx *ctx)
{
struct nvme_ns *ns = bdev->bd_disk->private_data;
struct nvme_ctrl *ctrl = ns->ctrl;
if (!ctrl->ops->init_dma_buf_io_ctx)
return -EINVAL;
return ctrl->ops->init_dma_buf_io_ctx(ctrl, ctx);
}
const struct block_device_operations nvme_bdev_ops = {
.owner = THIS_MODULE,
.ioctl = nvme_ioctl,
@ -2723,6 +2734,7 @@ const struct block_device_operations nvme_bdev_ops = {
.get_unique_id = nvme_get_unique_id,
.report_zones = nvme_report_zones,
.pr_ops = &nvme_pr_ops,
.init_dma_buf_io_ctx = nvme_init_dma_buf_io_ctx,
};
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,

2
drivers/nvme/host/nvme.h
View File

@ -669,6 +669,8 @@ struct nvme_ctrl_ops {
int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
void (*print_device_info)(struct nvme_ctrl *ctrl);
bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
int (*init_dma_buf_io_ctx)(struct nvme_ctrl *ctrl,
struct dma_buf_io_ctx *ctx);
unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool is_admin);
};

169
drivers/nvme/host/pci-dmabuf.c Normal file
View File

@ -0,0 +1,169 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Code for supporting dma-buf I/O maps in the NVMe driver.
* Copyright (C) 2026 Pavel Begunkov <asml.silence@gmail.com>
*/
#include <linux/dma-resv.h>
#include "pci-dmabuf.h"
static void nvme_dmabuf_invalidate_mappings(struct dma_buf_attachment *attach)
{
struct dma_buf_io_ctx *ctx = attach->importer_priv;
dma_buf_io_invalidate_mappings(ctx);
}
static const struct dma_buf_attach_ops nvme_dmabuf_importer_ops = {
.invalidate_mappings = nvme_dmabuf_invalidate_mappings,
.allow_peer2peer = true,
};
static struct dma_buf_io_map *nvme_dma_buf_io_map(struct dma_buf_io_ctx *ctx)
{
unsigned nr_entries = ctx->dmabuf->size / NVME_CTRL_PAGE_SIZE;
struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv;
struct dma_buf_attachment *attach = nvme_ctx->attach;
unsigned long tmp, i = 0;
struct nvme_dmabuf_map *map;
struct scatterlist *sg;
struct sg_table *sgt;
int ret;
dma_resv_assert_held(ctx->dmabuf->resv);
if (ctx->dmabuf->size % NVME_CTRL_PAGE_SIZE)
return ERR_PTR(-EINVAL);
map = kvmalloc_flex(*map, dma_list, nr_entries);
if (!map)
return ERR_PTR(-ENOMEM);
map->nr_entries = nr_entries;
sgt = dma_buf_map_attachment(attach, ctx->dir);
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
sgt = NULL;
goto err;
}
for_each_sgtable_dma_sg(sgt, sg, tmp) {
dma_addr_t dma_addr = sg_dma_address(sg);
unsigned long sg_len = sg_dma_len(sg);
if ((sg_len % NVME_CTRL_PAGE_SIZE) ||
(dma_addr % NVME_CTRL_PAGE_SIZE)) {
ret = -EINVAL;
goto err;
}
while (sg_len) {
map->dma_list[i++] = dma_addr;
dma_addr += NVME_CTRL_PAGE_SIZE;
sg_len -= NVME_CTRL_PAGE_SIZE;
}
}
ret = dma_buf_io_init_map(ctx, &map->base, sgt);
if (ret)
goto err;
map->sgt = sgt;
return &map->base;
err:
if (sgt)
dma_buf_unmap_attachment(attach, sgt, ctx->dir);
kvfree(map);
return ERR_PTR(ret);
}
static void nvme_dma_buf_io_unmap(struct dma_buf_io_ctx *ctx,
struct dma_buf_io_map *map_base)
{
struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv;
struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(map_base);
dma_resv_assert_held(ctx->dmabuf->resv);
dma_buf_unmap_attachment(nvme_ctx->attach, map->sgt, ctx->dir);
kvfree(map);
}
static void nvme_dma_buf_io_release(struct dma_buf_io_ctx *base_ctx)
{
struct nvme_dma_buf_io_ctx *ctx = base_ctx->dev_priv;
struct nvme_dma_buf_list *buf_list = ctx->buf_list;
mutex_lock(&buf_list->lock);
if (!list_empty(&ctx->list)) {
dma_buf_detach(ctx->base->dmabuf, ctx->attach);
list_del_init(&ctx->list);
}
mutex_unlock(&buf_list->lock);
nvme_put_ctrl(ctx->ctrl);
kfree(ctx);
}
void __nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list)
{
struct nvme_dma_buf_io_ctx *ctx, *tmp;
mutex_lock(&list->lock);
list->dead = true;
list_for_each_entry_safe(ctx, tmp, &list->ctxs, list) {
dma_buf_io_detach(ctx->base);
dma_buf_detach(ctx->base->dmabuf, ctx->attach);
list_del_init(&ctx->list);
}
mutex_unlock(&list->lock);
}
const struct dma_buf_io_ops nvme_dma_buf_io_ops = {
.map = nvme_dma_buf_io_map,
.unmap = nvme_dma_buf_io_unmap,
.release = nvme_dma_buf_io_release,
};
static int nvme_init_io_ctx_locked(struct nvme_ctrl *ctrl,
struct nvme_dma_buf_list *buf_list,
struct dma_buf_io_ctx *ctx)
{
struct nvme_dma_buf_io_ctx *nvme_ctx;
struct dma_buf_attachment *attach;
if (buf_list->dead)
return -ENODEV;
nvme_ctx = kzalloc_obj(*nvme_ctx);
if (!nvme_ctx)
return -ENOMEM;
attach = dma_buf_dynamic_attach(ctx->dmabuf, ctrl->dev,
&nvme_dmabuf_importer_ops, ctx);
if (IS_ERR(attach)) {
kfree(nvme_ctx);
return PTR_ERR(attach);
}
nvme_get_ctrl(ctrl);
list_add(&nvme_ctx->list, &buf_list->ctxs);
nvme_ctx->attach = attach;
nvme_ctx->base = ctx;
nvme_ctx->ctrl = ctrl;
nvme_ctx->buf_list = buf_list;
ctx->dev_priv = nvme_ctx;
ctx->dev_ops = &nvme_dma_buf_io_ops;
return 0;
}
int __nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl,
struct nvme_dma_buf_list *buf_list,
struct dma_buf_io_ctx *ctx)
{
int ret;
mutex_lock(&buf_list->lock);
ret = nvme_init_io_ctx_locked(ctrl, buf_list, ctx);
mutex_unlock(&buf_list->lock);
return ret;
}

75
drivers/nvme/host/pci-dmabuf.h Normal file
View File

@ -0,0 +1,75 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2026 Pavel Begunkov <asml.silence@gmail.com>
*/
#ifndef _NVME_PCI_DMABUF_H
#define _NVME_PCI_DMABUF_H
#include <linux/dma-buf-io.h>
#include "nvme.h"
struct nvme_dma_buf_list {
struct list_head ctxs;
struct mutex lock;
bool dead;
};
struct nvme_dma_buf_io_ctx {
struct dma_buf_attachment *attach;
struct nvme_dma_buf_list *buf_list;
struct dma_buf_io_ctx *base;
struct nvme_ctrl *ctrl;
struct list_head list;
};
struct nvme_dmabuf_map {
struct dma_buf_io_map base;
struct sg_table *sgt;
unsigned nr_entries;
dma_addr_t dma_list[] __counted_by(nr_entries);
};
static inline struct nvme_dmabuf_map *to_nvme_dmabuf_map(
struct dma_buf_io_map *map)
{
return container_of(map, struct nvme_dmabuf_map, base);
}
static inline struct nvme_dmabuf_map *nvme_req_to_map(struct request *req)
{
struct dma_buf_io_map *map = req->bio->bi_dmabuf_map;
return to_nvme_dmabuf_map(map);
}
void __nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list);
int __nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl,
struct nvme_dma_buf_list *buf_list,
struct dma_buf_io_ctx *ctx);
static inline void nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list)
{
if (IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
__nvme_pci_remove_dmabuf(list);
}
static inline int nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl,
struct nvme_dma_buf_list *buf_list,
struct dma_buf_io_ctx *ctx)
{
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return -EOPNOTSUPP;
return __nvme_init_dma_buf_io_ctx(ctrl, buf_list, ctx);
}
static inline void nvme_init_buf_list(struct nvme_dma_buf_list *buf_list)
{
INIT_LIST_HEAD(&buf_list->ctxs);
mutex_init(&buf_list->lock);
buf_list->dead = false;
}
#endif /* _NVME_PCI_DMABUF_H */

247
drivers/nvme/host/pci.c
View File

@ -30,6 +30,7 @@
#include "trace.h"
#include "nvme.h"
#include "pci-dmabuf.h"
#define SQ_SIZE(q) ((q)->q_depth << (q)->sqes)
#define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion))
@ -318,6 +319,7 @@ struct nvme_dev {
bool hmb;
struct sg_table *hmb_sgt;
mempool_t *dmavec_mempool;
struct nvme_dma_buf_list dmabufs;
/* shadow doorbell buffer support: */
__le32 *dbbuf_dbs;
@ -865,6 +867,76 @@ static void nvme_free_descriptors(struct request *req)
}
}
static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req,
struct nvme_queue *nvmeq)
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
struct nvme_dmabuf_map *map = nvme_req_to_map(req);
unsigned bvec_done = req->bio->bi_iter.bi_offset;
unsigned map_idx = bvec_done / NVME_CTRL_PAGE_SIZE;
unsigned offset = bvec_done & (NVME_CTRL_PAGE_SIZE - 1);
int length = blk_rq_payload_bytes(req) - (NVME_CTRL_PAGE_SIZE - offset);
dma_addr_t *dma_list = map->dma_list;
u64 prp1_dma = dma_list[map_idx++] + offset;
u64 dma_addr, prp2_dma;
dma_addr_t prp_dma;
__le64 *prp_list;
unsigned i;
if (length <= 0) {
prp2_dma = 0;
goto done;
}
if (length <= NVME_CTRL_PAGE_SIZE) {
prp2_dma = dma_list[map_idx];
goto done;
}
if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <=
NVME_SMALL_POOL_SIZE / sizeof(__le64))
iod->flags |= IOD_SMALL_DESCRIPTOR;
prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
&prp_dma);
if (!prp_list)
return BLK_STS_RESOURCE;
iod->descriptors[iod->nr_descriptors++] = prp_list;
prp2_dma = prp_dma;
i = 0;
for (;;) {
if (i == NVME_CTRL_PAGE_SIZE >> 3) {
__le64 *old_prp_list = prp_list;
prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large,
GFP_ATOMIC, &prp_dma);
if (!prp_list)
goto free_prps;
iod->descriptors[iod->nr_descriptors++] = prp_list;
prp_list[0] = old_prp_list[i - 1];
old_prp_list[i - 1] = cpu_to_le64(prp_dma);
i = 1;
}
dma_addr = dma_list[map_idx++];
prp_list[i++] = cpu_to_le64(dma_addr);
length -= NVME_CTRL_PAGE_SIZE;
if (length <= 0)
break;
}
done:
iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma);
iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma);
return BLK_STS_OK;
free_prps:
iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma);
iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma);
nvme_free_descriptors(req);
return BLK_STS_RESOURCE;
}
static void nvme_free_prps(struct request *req, unsigned int attrs)
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
@ -943,6 +1015,9 @@ static void nvme_unmap_data(struct request *req)
struct device *dma_dev = nvmeq->dev->dev;
unsigned int attrs = 0;
if (blk_mq_rq_is_dmabuf(req))
goto out_free_desc;
if (iod->flags & IOD_SINGLE_SEGMENT) {
static_assert(offsetof(union nvme_data_ptr, prp1) ==
offsetof(union nvme_data_ptr, sgl.addr));
@ -967,6 +1042,7 @@ static void nvme_unmap_data(struct request *req)
nvme_free_prps(req, attrs);
}
out_free_desc:
if (iod->nr_descriptors)
nvme_free_descriptors(req);
}
@ -1148,11 +1224,17 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req,
}
static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
dma_addr_t addr, u32 len)
{
sge->addr = cpu_to_le64(addr);
sge->length = cpu_to_le32(len);
sge->type = NVME_SGL_FMT_DATA_DESC << 4;
}
static void nvme_pci_dma_iter_set_sgl(struct nvme_sgl_desc *sge,
struct blk_dma_iter *iter)
{
sge->addr = cpu_to_le64(iter->addr);
sge->length = cpu_to_le32(iter->len);
sge->type = NVME_SGL_FMT_DATA_DESC << 4;
nvme_pci_sgl_set_data(sge, iter->addr, iter->len);
}
static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
@ -1163,6 +1245,142 @@ static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
}
static unsigned int nvme_pci_dmabuf_sgl_nents(struct request *req)
{
struct bio *bio = req->bio;
struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map);
struct scatterlist *sg;
unsigned long tmp;
size_t offset = bio->bi_iter.bi_offset;
size_t remaining = blk_rq_payload_bytes(req);
unsigned int nents = 0;
for_each_sgtable_dma_sg(map->sgt, sg, tmp) {
size_t sg_len = sg_dma_len(sg);
if (!remaining)
break;
if (offset >= sg_len) {
offset -= sg_len;
continue;
}
sg_len -= offset;
offset = 0;
do {
size_t chunk = min(remaining, sg_len);
nents++;
sg_len -= chunk;
remaining -= chunk;
} while (sg_len && remaining);
}
if (unlikely(remaining))
return 0;
return nents;
}
static blk_status_t nvme_rq_setup_dmabuf_sgl(struct request *req,
struct nvme_queue *nvmeq, unsigned int entries)
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
struct nvme_dmabuf_map *map = nvme_req_to_map(req);
size_t length = blk_rq_payload_bytes(req);
struct nvme_sgl_desc *sg_list = &iod->cmd.common.dptr.sgl;
size_t offset = req->bio->bi_iter.bi_offset;
bool pooled = entries > 1;
dma_addr_t sgl_dma = 0;
unsigned int mapped = 0;
unsigned long tmp;
struct scatterlist *sg;
size_t remaining = length;
if (!entries)
return BLK_STS_IOERR;
iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
iod->total_len = length;
/* entries == 1 fits in the inline descriptor; more needs a pool. */
if (pooled) {
if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list))
iod->flags |= IOD_SMALL_DESCRIPTOR;
sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
&sgl_dma);
if (!sg_list)
return BLK_STS_RESOURCE;
iod->descriptors[iod->nr_descriptors++] = sg_list;
}
for_each_sgtable_dma_sg(map->sgt, sg, tmp) {
size_t sg_len = sg_dma_len(sg);
dma_addr_t addr = sg_dma_address(sg);
if (!remaining)
break;
if (offset >= sg_len) {
offset -= sg_len;
continue;
}
addr += offset;
sg_len -= offset;
offset = 0;
do {
u32 chunk = min_t(size_t, remaining, sg_len);
if (WARN_ON_ONCE(mapped == entries))
goto err_free;
nvme_pci_sgl_set_data(&sg_list[mapped++], addr, chunk);
addr += chunk;
sg_len -= chunk;
remaining -= chunk;
} while (sg_len && remaining);
}
if (unlikely(remaining))
goto err_free;
if (pooled)
nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma,
mapped);
return BLK_STS_OK;
err_free:
if (pooled) {
iod->nr_descriptors--;
dma_pool_free(nvme_dma_pool(nvmeq, iod), sg_list, sgl_dma);
}
return BLK_STS_IOERR;
}
static blk_status_t nvme_rq_setup_dmabuf(struct request *req,
struct nvme_queue *nvmeq, enum nvme_use_sgl use_sgl)
{
unsigned int entries;
size_t avg_seg;
if (use_sgl == SGL_UNSUPPORTED)
return nvme_rq_setup_dmabuf_map(req, nvmeq);
entries = nvme_pci_dmabuf_sgl_nents(req);
if (use_sgl == SGL_FORCED)
return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries);
avg_seg = entries ?
DIV_ROUND_UP(blk_rq_payload_bytes(req), entries) : 0;
if (sgl_threshold && avg_seg >= sgl_threshold)
return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries);
return nvme_rq_setup_dmabuf_map(req, nvmeq);
}
static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
struct blk_dma_iter *iter)
{
@ -1177,7 +1395,7 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) {
nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter);
nvme_pci_dma_iter_set_sgl(&iod->cmd.common.dptr.sgl, iter);
iod->total_len += iter->len;
return BLK_STS_OK;
}
@ -1199,7 +1417,7 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
iter->status = BLK_STS_IOERR;
break;
}
nvme_pci_sgl_set_data(&sg_list[mapped++], iter);
nvme_pci_dma_iter_set_sgl(&sg_list[mapped++], iter);
iod->total_len += iter->len;
} while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter));
@ -1257,6 +1475,9 @@ static blk_status_t nvme_map_data(struct request *req)
struct blk_dma_iter iter;
blk_status_t ret;
if (blk_mq_rq_is_dmabuf(req))
return nvme_rq_setup_dmabuf(req, nvmeq, use_sgl);
/*
* Try to skip the DMA iterator for single segment requests, as that
* significantly improves performances for small I/O sizes.
@ -1359,13 +1580,13 @@ static blk_status_t nvme_pci_setup_meta_iter(struct request *req)
iod->cmd.common.metadata = cpu_to_le64(sgl_dma);
if (entries == 1) {
iod->meta_total_len = iter.len;
nvme_pci_sgl_set_data(sg_list, &iter);
nvme_pci_dma_iter_set_sgl(sg_list, &iter);
return BLK_STS_OK;
}
sgl_dma += sizeof(*sg_list);
do {
nvme_pci_sgl_set_data(&sg_list[++i], &iter);
nvme_pci_dma_iter_set_sgl(&sg_list[++i], &iter);
iod->meta_total_len += iter.len;
} while (blk_rq_integrity_dma_map_iter_next(req, dev->dev, &iter));
@ -3564,6 +3785,14 @@ static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl,
return 0;
}
static int nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl,
struct dma_buf_io_ctx *ctx)
{
struct nvme_dev *dev = to_nvme_dev(ctrl);
return nvme_init_dma_buf_io_ctx(ctrl, &dev->dmabufs, ctx);
}
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.name = "pcie",
.module = THIS_MODULE,
@ -3579,6 +3808,7 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.print_device_info = nvme_pci_print_device_info,
.supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma,
.get_virt_boundary = nvme_pci_get_virt_boundary,
.init_dma_buf_io_ctx = nvme_pci_init_dma_buf_io_ctx,
};
static int nvme_dev_map(struct nvme_dev *dev)
@ -3701,6 +3931,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
return ERR_PTR(-ENOMEM);
INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
mutex_init(&dev->shutdown_lock);
nvme_init_buf_list(&dev->dmabufs);
dev->nr_write_queues = write_queues;
dev->nr_poll_queues = poll_queues;
@ -3916,6 +4147,7 @@ static void nvme_remove(struct pci_dev *pdev)
nvme_stop_ctrl(&dev->ctrl);
nvme_remove_namespaces(&dev->ctrl);
nvme_dev_disable(dev, true);
nvme_pci_remove_dmabuf(&dev->dmabufs);
nvme_free_host_mem(dev);
nvme_dev_remove_admin(dev);
nvme_dbbuf_dma_free(dev);
@ -4346,5 +4578,6 @@ MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
MODULE_VERSION("1.0");
MODULE_DESCRIPTION("NVMe host PCIe transport driver");
MODULE_IMPORT_NS("DMA_BUF");
module_init(nvme_init);
module_exit(nvme_exit);

21
include/linux/bio.h
View File

@ -80,7 +80,8 @@ static inline bool bio_no_advance_iter(const struct bio *bio)
{
return bio_op(bio) == REQ_OP_DISCARD ||
bio_op(bio) == REQ_OP_SECURE_ERASE ||
bio_op(bio) == REQ_OP_WRITE_ZEROES;
bio_op(bio) == REQ_OP_WRITE_ZEROES ||
op_is_dmabuf(bio->bi_opf);
}
static inline void *bio_data(struct bio *bio)
@ -113,11 +114,13 @@ static inline void bio_advance_iter(const struct bio *bio,
{
iter->bi_sector += bytes >> 9;
if (bio_no_advance_iter(bio))
if (bio_no_advance_iter(bio)) {
iter->bi_size -= bytes;
else
iter->bi_offset += bytes;
} else {
bvec_iter_advance(bio->bi_io_vec, iter, bytes);
/* TODO: It is reasonable to complete bio with error here. */
}
}
/* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
@ -127,10 +130,12 @@ static inline void bio_advance_iter_single(const struct bio *bio,
{
iter->bi_sector += bytes >> 9;
if (bio_no_advance_iter(bio))
if (bio_no_advance_iter(bio)) {
iter->bi_size -= bytes;
else
iter->bi_offset += bytes;
} else {
bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
}
}
void __bio_advance(struct bio *, unsigned bytes);
@ -434,12 +439,12 @@ static inline void bio_wouldblock_error(struct bio *bio)
/*
* Calculate number of bvec segments that should be allocated to fit data
* pointed by @iter. If @iter is backed by bvec it's going to be reused
* instead of allocating a new one.
* pointed by @iter. If @iter is backed by a bvec or a dmabuf, the bvec array /
* the dma map are going to be reused, and so no extra allocation is required.
*/
static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
{
if (iov_iter_is_bvec(iter))
if (iov_iter_is_bvec(iter) || iov_iter_is_dmabuf_map(iter))
return 0;
return iov_iter_npages(iter, max_segs);
}

7
include/linux/blk-mq.h
View File

@ -1014,6 +1014,13 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)
return rq + 1;
}
static inline bool blk_mq_rq_is_dmabuf(struct request *rq)
{
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return false;
return rq->bio && op_is_dmabuf(rq->bio->bi_opf);
}
static inline struct blk_mq_hw_ctx *queue_hctx(struct request_queue *q, int id)
{
struct blk_mq_hw_ctx *hctx;

14
include/linux/blk_types.h
View File

@ -233,7 +233,12 @@ struct bio {
atomic_t __bi_remaining;
/* The actual vec list, preserved by bio_reset() */
struct bio_vec *bi_io_vec;
union {
struct bio_vec *bi_io_vec;
/* Driver specific dma map, valid IFF REQ_DMABUF is set */
struct dma_buf_io_map *bi_dmabuf_map;
};
struct bvec_iter bi_iter;
union {
@ -402,6 +407,7 @@ enum req_flag_bits {
__REQ_DRV, /* for driver use */
__REQ_FS_PRIVATE, /* for file system (submitter) use */
__REQ_ATOMIC, /* for atomic write operations */
__REQ_DMABUF, /* Using premmaped dma buffers */
/*
* Command specific flags, keep last:
*/
@ -434,6 +440,7 @@ enum req_flag_bits {
#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV)
#define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)
#define REQ_ATOMIC (__force blk_opf_t)(1ULL << __REQ_ATOMIC)
#define REQ_DMABUF (__force blk_opf_t)(1ULL << __REQ_DMABUF)
#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
@ -487,6 +494,11 @@ static inline bool op_is_discard(blk_opf_t op)
return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
}
static inline bool op_is_dmabuf(blk_opf_t op)
{
return op & REQ_DMABUF;
}
/*
* Check if a bio or request operation is a zone management operation.
*/

2
include/linux/blkdev.h
View File

@ -1599,6 +1599,8 @@ struct block_device_operations {
/* returns the length of the identifier or a negative errno: */
int (*get_unique_id)(struct gendisk *disk, u8 id[16],
enum blk_unique_id id_type);
int (*init_dma_buf_io_ctx)(struct block_device *,
struct dma_buf_io_ctx *);
struct module *owner;
const struct pr_ops *pr_ops;

3
include/linux/bvec.h
View File

@ -108,7 +108,8 @@ struct bvec_iter {
unsigned int bi_idx;
/*
* Current offset in the bvec entry pointed to by `bi_idx`.
* Current offset in the bvec entry pointed to by `bi_idx` or into
* a dma-buf map.
*/
unsigned int bi_offset;
} __packed __aligned(4);

126
include/linux/dma-buf-io.h Normal file
View File

@ -0,0 +1,126 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __DMA_BUF_IO_H__
#define __DMA_BUF_IO_H__
#include <linux/dma-buf.h>
struct dma_buf_io_ctx;
struct dma_buf_io_map;
struct dma_buf_io_ops {
/*
* Create a new map for the given ctx. Called with the reservation
* lock held.
*/
struct dma_buf_io_map *(*map)(struct dma_buf_io_ctx *ctx);
/*
* Clean up device specific parts of the @map. Called with the
* reservation lock held.
*/
void (*unmap)(struct dma_buf_io_ctx *ctx, struct dma_buf_io_map *map);
/*
* The user tries to destroy the ctx. Release all device specific
* parts of the token.
*/
void (*release)(struct dma_buf_io_ctx *);
};
struct dma_buf_io_map {
/*
* Counts attached requests and other users. Device specific unmapping
* is deferred until all refs are dropped.
*/
struct percpu_ref refs;
/*
* Shift for the minimum segment size of the mapping.
*/
unsigned min_seg_shift;
struct dma_buf_io_ctx *ctx;
struct completion drained;
};
struct dma_buf_io_ctx {
struct dma_buf_io_map __rcu *map;
struct dma_buf *dmabuf;
enum dma_data_direction dir;
bool is_dead;
void *dev_priv;
const struct dma_buf_io_ops *dev_ops;
};
int dma_buf_io_ctx_create(struct file *file,
struct dma_buf *dmabuf,
enum dma_data_direction dir,
struct dma_buf_io_ctx **ctx);
void dma_buf_io_ctx_release(struct dma_buf_io_ctx *ctx);
struct dma_buf_io_map *dma_buf_io_create_map(struct dma_buf_io_ctx *ctx);
/* Internal helper, see dma_buf_io_get_map() */
static inline struct dma_buf_io_map *
__dma_buf_io_get_map(struct dma_buf_io_ctx *ctx)
{
struct dma_buf_io_map *map;
guard(rcu)();
map = rcu_dereference(ctx->map);
if (unlikely(!map || !percpu_ref_tryget_live_rcu(&map->refs)))
return NULL;
return map;
}
/**
* dma_buf_io_get_map - Create a new or reference an existing map.
* @ctx: [in] dma-buf I/O context
* @nowait: [in] specifies whether it's allowed to block
*
* Create a new or acquire an existing map. On success, it returns
* a referenced map, which should eventually be put down with
* dma_buf_io_map_drop(). If @nowait is set and the function would block,
* it returns -EAGAIN, and the caller is expected to retry from a context
* where blocking is allowed.
*
* Returns a map on success, or ERR_PTR encoded errno otherwise.
*/
static inline struct dma_buf_io_map *
dma_buf_io_get_map(struct dma_buf_io_ctx *ctx, bool nowait)
{
struct dma_buf_io_map *map;
map = __dma_buf_io_get_map(ctx);
if (likely(map))
return map;
if (nowait)
return ERR_PTR(-EAGAIN);
return dma_buf_io_create_map(ctx);
}
/**
* dma_buf_io_map_drop - Drop the map.
* @map: [in] dma-buf I/O map
*
* Put down a reference to the @map acquired via dma_buf_io_get_map().
* Should be called as soon as I/O is completed.
*/
static inline void dma_buf_io_map_drop(struct dma_buf_io_map *map)
{
percpu_ref_put(&map->refs);
}
/*
* Device API
*/
void dma_buf_io_invalidate_mappings(struct dma_buf_io_ctx *ctx);
int dma_buf_io_init_map(struct dma_buf_io_ctx *ctx, struct dma_buf_io_map *map,
struct sg_table *sgt);
void dma_buf_io_detach(struct dma_buf_io_ctx *ctx);
#endif

2
include/linux/fs.h
View File

@ -1914,6 +1914,7 @@ struct dir_context {
#define COPY_FILE_SPLICE (1 << 0)
struct io_uring_cmd;
struct dma_buf_io_ctx;
struct offset_ctx;
struct file_operations {
@ -1960,6 +1961,7 @@ struct file_operations {
int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
unsigned int poll_flags);
int (*mmap_prepare)(struct vm_area_desc *);
int (*init_dma_buf_io_ctx)(struct file *, struct dma_buf_io_ctx *);
} __randomize_layout;
/* Supports async buffered reads */

5
include/linux/io_uring_types.h
View File

@ -11,6 +11,7 @@
struct iou_loop_params;
struct io_uring_bpf_ops;
struct dma_buf_io_map;
enum {
/*
@ -617,6 +618,7 @@ enum {
REQ_F_IMPORT_BUFFER_BIT,
REQ_F_SQE_COPIED_BIT,
REQ_F_IOPOLL_BIT,
REQ_F_DMABUF_BIT,
/* not a real bit, just to check we're not overflowing the space */
__REQ_F_LAST_BIT,
@ -712,6 +714,8 @@ enum {
REQ_F_SQE_COPIED = IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT),
/* request must be iopolled to completion (set in ->issue()) */
REQ_F_IOPOLL = IO_REQ_FLAG(REQ_F_IOPOLL_BIT),
/* there is a dma map attached to request that needs to be dropped */
REQ_F_DMABUF = IO_REQ_FLAG(REQ_F_DMABUF_BIT),
};
struct io_tw_req {
@ -834,6 +838,7 @@ struct io_kiocb {
/* custom credentials, valid IFF REQ_F_CREDS is set */
const struct cred *creds;
struct io_wq_work work;
struct dma_buf_io_map *dmabuf_map;
struct io_big_cqe {
u64 extra1;

2
include/linux/iov_iter.h
View File

@ -310,6 +310,8 @@ size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
return iterate_folioq(iter, len, priv, priv2, step);
if (iov_iter_is_xarray(iter))
return iterate_xarray(iter, len, priv, priv2, step);
if (WARN_ON_ONCE(iov_iter_is_dmabuf_map(iter)))
return 0;
return iterate_discard(iter, len, priv, priv2, step);
}

11
include/linux/uio.h
View File

@ -12,6 +12,7 @@
struct page;
struct folio_queue;
struct dma_buf_io_map;
typedef unsigned int __bitwise iov_iter_extraction_t;
@ -29,6 +30,7 @@ enum iter_type {
ITER_FOLIOQ,
ITER_XARRAY,
ITER_DISCARD,
ITER_DMABUF_MAP,
};
#define ITER_SOURCE 1 // == WRITE
@ -71,6 +73,7 @@ struct iov_iter {
const struct folio_queue *folioq;
struct xarray *xarray;
void __user *ubuf;
struct dma_buf_io_map *dmabuf_map;
};
size_t count;
};
@ -155,6 +158,11 @@ static inline bool iov_iter_is_xarray(const struct iov_iter *i)
return iov_iter_type(i) == ITER_XARRAY;
}
static inline bool iov_iter_is_dmabuf_map(const struct iov_iter *i)
{
return iov_iter_type(i) == ITER_DMABUF_MAP;
}
static inline unsigned char iov_iter_rw(const struct iov_iter *i)
{
return i->data_source ? WRITE : READ;
@ -300,6 +308,9 @@ void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction,
unsigned int first_slot, unsigned int offset, size_t count);
void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
loff_t start, size_t count);
void iov_iter_dmabuf_map(struct iov_iter *i, unsigned int direction,
struct dma_buf_io_map *map,
loff_t off, size_t count);
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
size_t maxsize, unsigned maxpages, size_t *start);
ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages,

31
include/uapi/linux/io_uring.h
View File

@ -791,13 +791,42 @@ struct io_uring_rsrc_update {
struct io_uring_rsrc_update2 {
__u32 offset;
__u32 resv;
__u32 flags;
__aligned_u64 data;
__aligned_u64 tags;
__u32 nr;
__u32 resv2;
};
/* struct io_uring_rsrc_update2::flags */
enum io_uring_rsrc_reg_flags {
/*
* Use the extended descriptor format for buffer updates,
* see struct io_uring_regbuf_desc
*/
IORING_RSRC_UPDATE_EXTENDED = 1U << 1,
};
/* Buffer registration type, passed in struct io_uring_regbuf_desc::type */
enum io_uring_regbuf_type {
IO_REGBUF_TYPE_EMPTY,
IO_REGBUF_TYPE_UADDR,
IO_REGBUF_TYPE_DMABUF,
__IO_REGBUF_TYPE_MAX,
};
struct io_uring_regbuf_desc {
__u32 type; /* enum io_uring_regbuf_type */
__u32 flags;
__u64 size;
__u64 uaddr;
__s32 dmabuf_fd;
__s32 target_fd;
__u64 __resv[6];
};
/* Skip updating fd indexes set to this value in the fd table */
#define IORING_REGISTER_FILES_SKIP (-2)

1
io_uring/Makefile
View File

@ -26,3 +26,4 @@ obj-$(CONFIG_PROC_FS) += fdinfo.o
obj-$(CONFIG_IO_URING_MOCK_FILE) += mock_file.o
obj-$(CONFIG_IO_URING_BPF) += bpf_filter.o
obj-$(CONFIG_IO_URING_BPF_OPS) += bpf-ops.o
obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf.o

71
io_uring/dma-buf.c Normal file
View File

@ -0,0 +1,71 @@
#include "dma-buf.h"
static inline void io_release_dmabuf(void *priv)
{
struct io_buf_dma *bd = priv;
fput(bd->target_file);
dma_buf_io_ctx_release(bd->ctx);
kfree(bd);
}
int __io_register_dmabuf(struct io_ring_ctx *ctx,
struct io_uring_regbuf_desc *desc,
struct io_mapped_ubuf *imu)
{
struct io_buf_dma *bd = NULL;
struct file *target_file = NULL;
struct dma_buf *dmabuf = NULL;
int ret;
/*
* We have to drop the dma-buf map early when IO is finished. Disable
* IOPOLL for now as we can't wait potentially infintely for a task
* to do that.
*/
if (ctx->flags & IORING_SETUP_IOPOLL)
return -EOPNOTSUPP;
if (desc->uaddr || desc->size)
return -EINVAL;
bd = kzalloc(sizeof(*bd), GFP_KERNEL);
if (!bd)
return -ENOMEM;
target_file = fget(desc->target_fd);
if (!target_file) {
ret = -EBADF;
goto err;
}
dmabuf = dma_buf_get(desc->dmabuf_fd);
if (IS_ERR(dmabuf)) {
ret = PTR_ERR(dmabuf);
dmabuf = NULL;
goto err;
}
ret = io_validate_user_buf_range(0, dmabuf->size);
if (ret)
goto err;
ret = dma_buf_io_ctx_create(target_file, dmabuf, DMA_BIDIRECTIONAL,
&bd->ctx);
if (ret)
goto err;
bd->target_file = target_file;
imu->len = dmabuf->size;
imu->release = io_release_dmabuf;
imu->priv = bd;
imu->flags = IO_REGBUF_F_DMABUF;
imu->dir = IO_BUF_DEST | IO_BUF_SOURCE;
dma_buf_put(dmabuf);
return 0;
err:
kfree(bd);
if (target_file)
fput(target_file);
if (dmabuf)
dma_buf_put(dmabuf);
return ret;
}

76
io_uring/dma-buf.h Normal file
View File

@ -0,0 +1,76 @@
// SPDX-License-Identifier: GPL-2.0
#ifndef IOU_DMA_BUF_H
#define IOU_DMA_BUF_H
#include <linux/dma-buf-io.h>
#include <linux/io_uring_types.h>
#include "rsrc.h"
struct io_buf_dma {
struct dma_buf_io_ctx *ctx;
struct file *target_file;
};
int __io_register_dmabuf(struct io_ring_ctx *ctx,
struct io_uring_regbuf_desc *desc,
struct io_mapped_ubuf *imu);
static inline void io_detach_dmabuf_map(struct io_kiocb *req)
{
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return;
if (!(req->flags & REQ_F_DMABUF) || !req->dmabuf_map)
return;
dma_buf_io_map_drop(req->dmabuf_map);
req->dmabuf_map = NULL;
}
static inline int io_attach_dmabuf_map(struct io_kiocb *req,
struct iov_iter *iter,
unsigned issue_flags)
{
bool nowait = issue_flags & IO_URING_F_NONBLOCK;
struct dma_buf_io_map *map;
struct io_buf_dma *bd;
if (!(req->flags & REQ_F_DMABUF))
return 0;
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return -EOPNOTSUPP;
bd = req->buf_node->buf->priv;
map = dma_buf_io_get_map(bd->ctx, nowait);
if (unlikely(IS_ERR(map)))
return PTR_ERR(map);
req->dmabuf_map = map;
iter->dmabuf_map = map;
return 0;
}
static inline int io_import_dmabuf_map(struct io_kiocb *req,
int ddir, struct iov_iter *iter,
struct io_mapped_ubuf *imu,
size_t len, size_t offset)
{
struct io_buf_dma *bd = imu->priv;
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return -EOPNOTSUPP;
if (req->file != bd->target_file)
return -EBADF;
req->flags |= REQ_F_DMABUF;
iov_iter_dmabuf_map(iter, ddir, NULL, offset, len);
return 0;
}
static inline int io_register_dmabuf(struct io_ring_ctx *ctx,
struct io_uring_regbuf_desc *desc,
struct io_mapped_ubuf *imu)
{
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return -EOPNOTSUPP;
return __io_register_dmabuf(ctx, desc, imu);
}
#endif /* IOU_DMA_BUF_H */

178
io_uring/rsrc.c
View File

@ -10,6 +10,7 @@
#include <linux/compat.h>
#include <linux/io_uring.h>
#include <linux/io_uring/cmd.h>
#include <linux/dma-buf-io.h>
#include <uapi/linux/io_uring.h>
@ -19,6 +20,7 @@
#include "rsrc.h"
#include "memmap.h"
#include "register.h"
#include "dma-buf.h"
struct io_rsrc_update {
struct file *file;
@ -28,7 +30,7 @@ struct io_rsrc_update {
};
static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
struct iovec *iov);
struct io_uring_regbuf_desc *desc);
static int hpage_acct_ref(struct io_ring_ctx *ctx, struct page *hpage,
bool *acct_new)
@ -81,6 +83,18 @@ static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage)
#define IO_CACHED_BVECS_SEGS 32
static void io_iov_to_regbuf_desc(const struct iovec *iov,
struct io_uring_regbuf_desc *desc)
{
*desc = (struct io_uring_regbuf_desc) {
.type = IO_REGBUF_TYPE_UADDR,
.uaddr = (u64)(uintptr_t)iov->iov_base,
.size = iov->iov_len,
};
if (!desc->uaddr)
desc->type = IO_REGBUF_TYPE_EMPTY;
}
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
unsigned long page_limit, cur_pages, new_pages;
@ -326,6 +340,8 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
return -ENXIO;
if (up->offset + nr_args > ctx->file_table.data.nr)
return -EINVAL;
if (up->flags)
return -EINVAL;
for (done = 0; done < nr_args; done++) {
u64 tag = 0;
@ -385,9 +401,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
struct io_uring_rsrc_update2 *up,
unsigned int nr_args)
{
bool extended = up->flags & IORING_RSRC_UPDATE_EXTENDED;
u64 __user *tags = u64_to_user_ptr(up->tags);
struct iovec fast_iov, *iov;
struct iovec __user *uvec;
u64 user_data = up->data;
__u32 done;
int i, err;
@ -396,26 +411,49 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
return -ENXIO;
if (up->offset + nr_args > ctx->buf_table.nr)
return -EINVAL;
if (up->flags & ~IORING_RSRC_UPDATE_EXTENDED)
return -EINVAL;
for (done = 0; done < nr_args; done++) {
struct io_uring_regbuf_desc desc;
struct io_rsrc_node *node;
u64 tag = 0;
uvec = u64_to_user_ptr(user_data);
iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
if (IS_ERR(iov)) {
err = PTR_ERR(iov);
break;
}
if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
err = -EFAULT;
break;
}
node = io_sqe_buffer_register(ctx, iov);
if (extended) {
if (copy_from_user(&desc, u64_to_user_ptr(user_data),
sizeof(desc))) {
err = -EFAULT;
break;
}
user_data += sizeof(desc);
} else {
struct iovec __user *uvec = u64_to_user_ptr(user_data);
struct iovec fast_iov, *iov;
if (io_is_compat(ctx))
user_data += sizeof(struct compat_iovec);
else
user_data += sizeof(struct iovec);
iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
if (IS_ERR(iov)) {
err = PTR_ERR(iov);
break;
}
io_iov_to_regbuf_desc(iov, &desc);
}
node = io_sqe_buffer_register(ctx, &desc);
if (IS_ERR(node)) {
err = PTR_ERR(node);
break;
}
if (tag) {
if (!node) {
err = -EINVAL;
@ -426,10 +464,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
i = array_index_nospec(up->offset + done, ctx->buf_table.nr);
io_reset_rsrc_node(ctx, &ctx->buf_table, i);
ctx->buf_table.nodes[i] = node;
if (io_is_compat(ctx))
user_data += sizeof(struct compat_iovec);
else
user_data += sizeof(struct iovec);
}
return done ? done : err;
}
@ -464,7 +498,7 @@ int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
memset(&up, 0, sizeof(up));
if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
return -EFAULT;
if (up.resv || up.resv2)
if (up.resv2)
return -EINVAL;
return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
}
@ -478,7 +512,7 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
return -EINVAL;
if (copy_from_user(&up, arg, sizeof(up)))
return -EFAULT;
if (!up.nr || up.resv || up.resv2)
if (!up.nr || up.resv2)
return -EINVAL;
return __io_register_rsrc_update(ctx, type, &up, up.nr);
}
@ -578,12 +612,9 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
struct io_uring_rsrc_update2 up2;
int ret;
memset(&up2, 0, sizeof(up2));
up2.offset = up->offset;
up2.data = up->arg;
up2.nr = 0;
up2.tags = 0;
up2.resv = 0;
up2.resv2 = 0;
if (up->offset == IORING_FILE_INDEX_ALLOC) {
ret = io_files_update_with_index_alloc(req, issue_flags);
@ -869,27 +900,72 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages,
return true;
}
static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
struct iovec *iov)
static struct io_rsrc_node *io_register_dmabuf_desc(struct io_ring_ctx *ctx,
struct io_uring_regbuf_desc *desc)
{
struct io_rsrc_node *node = NULL;
struct io_mapped_ubuf *imu = NULL;
int ret;
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!node)
return ERR_PTR(-ENOMEM);
imu = io_alloc_imu(ctx, 0);
if (!imu) {
ret = -ENOMEM;
goto err;
}
memset(imu, 0, sizeof(*imu));
refcount_set(&imu->refs, 1);
ret = io_register_dmabuf(ctx, desc, imu);
if (ret)
goto err;
node->buf = imu;
return node;
err:
if (imu)
io_free_imu(ctx, imu);
if (node)
io_cache_free(&ctx->node_cache, node);
return ERR_PTR(ret);
}
static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
struct io_uring_regbuf_desc *desc)
{
unsigned long uaddr = (unsigned long)desc->uaddr;
size_t size = desc->size;
struct io_mapped_ubuf *imu = NULL;
struct page **pages = NULL;
struct io_rsrc_node *node;
unsigned long off;
size_t size;
int ret, nr_pages, i;
struct io_imu_folio_data data;
bool coalesced = false;
if (!iov->iov_base) {
if (iov->iov_len)
if (uaddr != desc->uaddr || size != desc->size)
return ERR_PTR(-EOVERFLOW);
if (desc->type >= __IO_REGBUF_TYPE_MAX)
return ERR_PTR(-EINVAL);
if (!mem_is_zero(&desc->__resv, sizeof(desc->__resv)) || desc->flags)
return ERR_PTR(-EINVAL);
if (desc->type == IO_REGBUF_TYPE_DMABUF)
return io_register_dmabuf_desc(ctx, desc);
if (desc->dmabuf_fd || desc->target_fd)
return ERR_PTR(-EINVAL);
if (desc->type == IO_REGBUF_TYPE_EMPTY) {
if (uaddr || size)
return ERR_PTR(-EFAULT);
/* remove the buffer without installing a new one */
return NULL;
}
ret = io_validate_user_buf_range((unsigned long)iov->iov_base,
iov->iov_len);
ret = io_validate_user_buf_range(uaddr, size);
if (ret)
return ERR_PTR(ret);
@ -898,8 +974,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
return ERR_PTR(-ENOMEM);
ret = -ENOMEM;
pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
&nr_pages);
pages = io_pin_pages(uaddr, size, &nr_pages);
if (IS_ERR(pages)) {
ret = PTR_ERR(pages);
pages = NULL;
@ -921,10 +996,9 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
if (ret)
goto done;
size = iov->iov_len;
/* store original address for later verification */
imu->ubuf = (unsigned long) iov->iov_base;
imu->len = iov->iov_len;
imu->ubuf = uaddr;
imu->len = size;
imu->folio_shift = PAGE_SHIFT;
imu->release = io_release_ubuf;
imu->priv = imu;
@ -934,7 +1008,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
imu->folio_shift = data.folio_shift;
refcount_set(&imu->refs, 1);
off = (unsigned long)iov->iov_base & ~PAGE_MASK;
off = uaddr & ~PAGE_MASK;
if (coalesced)
off += data.first_folio_page_idx << PAGE_SHIFT;
@ -986,6 +1060,7 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
memset(iov, 0, sizeof(*iov));
for (i = 0; i < nr_args; i++) {
struct io_uring_regbuf_desc desc;
struct io_rsrc_node *node;
u64 tag = 0;
@ -1009,7 +1084,8 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
}
}
node = io_sqe_buffer_register(ctx, iov);
io_iov_to_regbuf_desc(iov, &desc);
node = io_sqe_buffer_register(ctx, &desc);
if (IS_ERR(node)) {
ret = PTR_ERR(node);
break;
@ -1206,9 +1282,12 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,
return 0;
}
static int io_import_fixed(int ddir, struct iov_iter *iter,
static int io_import_fixed(struct io_kiocb *req,
int ddir, struct iov_iter *iter,
struct io_mapped_ubuf *imu,
u64 buf_addr, size_t len)
u64 buf_addr, size_t len,
unsigned issue_flags,
unsigned import_flags)
{
const struct bio_vec *bvec;
size_t folio_mask;
@ -1228,6 +1307,11 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
offset = buf_addr - imu->ubuf;
if (imu->flags & IO_REGBUF_F_DMABUF) {
if (!(import_flags & IO_REGBUF_IMPORT_ALLOW_DMABUF))
return -EFAULT;
return io_import_dmabuf_map(req, ddir, iter, imu, len, offset);
}
if (imu->flags & IO_REGBUF_F_KBUF)
return io_import_kbuf(ddir, iter, imu, len, offset);
@ -1281,16 +1365,17 @@ inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
return NULL;
}
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
int __io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags)
unsigned issue_flags, unsigned import_flags)
{
struct io_rsrc_node *node;
node = io_find_buf_node(req, issue_flags);
if (!node)
return -EFAULT;
return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
return io_import_fixed(req, ddir, iter, node->buf, buf_addr, len,
issue_flags, import_flags);
}
static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
@ -1422,6 +1507,13 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
if (!src_node) {
dst_node = NULL;
} else {
/* we don't allow IOPOLL rw with dma-buf for now */
if ((src_node->buf->flags & IO_REGBUF_F_DMABUF) &&
(ctx->flags & IORING_SETUP_IOPOLL)) {
io_rsrc_data_free(ctx, &data);
return -EOPNOTSUPP;
}
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!dst_node) {
io_rsrc_data_free(ctx, &data);
@ -1692,9 +1784,9 @@ static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,
return 0;
}
int io_import_reg_vec(int ddir, struct iov_iter *iter,
int __io_import_reg_vec(int ddir, struct iov_iter *iter,
struct io_kiocb *req, struct iou_vec *vec,
unsigned nr_iovs, unsigned issue_flags)
unsigned nr_iovs, unsigned issue_flags, unsigned import_flags)
{
struct io_rsrc_node *node;
struct io_mapped_ubuf *imu;
@ -1712,7 +1804,9 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,
iovec_off = vec->nr - nr_iovs;
iov = vec->iovec + iovec_off;
if (imu->flags & IO_REGBUF_F_KBUF) {
if (imu->flags & IO_REGBUF_F_DMABUF) {
return -EOPNOTSUPP;
} else if (imu->flags & IO_REGBUF_F_KBUF) {
int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
if (unlikely(ret))

31
io_uring/rsrc.h
View File

@ -26,7 +26,12 @@ struct io_rsrc_node {
};
enum {
IO_REGBUF_F_KBUF = 1,
IO_REGBUF_F_KBUF = 1 << 0,
IO_REGBUF_F_DMABUF = 1 << 1,
};
enum {
IO_REGBUF_IMPORT_ALLOW_DMABUF = 1 << 1,
};
struct io_mapped_ubuf {
@ -61,12 +66,28 @@ int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
unsigned issue_flags);
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
int __io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags);
int io_import_reg_vec(int ddir, struct iov_iter *iter,
unsigned issue_flags, unsigned import_flags);
int __io_import_reg_vec(int ddir, struct iov_iter *iter,
struct io_kiocb *req, struct iou_vec *vec,
unsigned nr_iovs, unsigned issue_flags);
unsigned nr_iovs, unsigned issue_flags,
unsigned import_flags);
static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags)
{
return __io_import_reg_buf(req, iter, buf_addr, len, ddir, issue_flags, 0);
}
static inline int io_import_reg_vec(int ddir, struct iov_iter *iter,
struct io_kiocb *req, struct iou_vec *vec,
unsigned nr_iovs, unsigned issue_flags)
{
return __io_import_reg_vec(ddir, iter, req, vec, nr_iovs, issue_flags, 0);
}
int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
const struct iovec __user *uvec, size_t uvec_segs);

33
io_uring/rw.c
View File

@ -23,6 +23,7 @@
#include "rsrc.h"
#include "poll.h"
#include "rw.h"
#include "dma-buf.h"
static void io_complete_rw(struct kiocb *kiocb, long res);
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res);
@ -358,13 +359,18 @@ static int io_init_rw_fixed(struct io_kiocb *req, unsigned int issue_flags,
struct io_async_rw *io = req->async_data;
int ret;
if (io->bytes_done)
return 0;
if (!io->bytes_done) {
ret = __io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir,
issue_flags, IO_REGBUF_IMPORT_ALLOW_DMABUF);
if (ret)
return ret;
iov_iter_save_state(&io->iter, &io->iter_state);
}
ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir,
issue_flags);
iov_iter_save_state(&io->iter, &io->iter_state);
return ret;
ret = io_attach_dmabuf_map(req, &io->iter, issue_flags);
if (ret)
return ret;
return 0;
}
int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@ -577,6 +583,8 @@ static void io_complete_rw(struct kiocb *kiocb, long res)
struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
struct io_kiocb *req = cmd_to_io_kiocb(rw);
io_detach_dmabuf_map(req);
/* ring owner may block in freeze_super() before task_work runs */
if (kiocb->ki_flags & IOCB_WRITE)
io_req_end_write(req);
@ -680,7 +688,7 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)
!(kiocb->ki_filp->f_flags & O_NONBLOCK))
return -EAGAIN;
if ((req->flags & REQ_F_BUF_NODE) &&
(req->buf_node->buf->flags & IO_REGBUF_F_KBUF))
(req->buf_node->buf->flags & (IO_REGBUF_F_KBUF|IO_REGBUF_F_DMABUF)))
return -EFAULT;
ppos = io_kiocb_ppos(kiocb);
@ -1208,7 +1216,10 @@ int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(ret))
return ret;
return io_read(req, issue_flags);
ret = io_read(req, issue_flags);
if (ret != -EIOCBQUEUED)
io_detach_dmabuf_map(req);
return ret;
}
int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags)
@ -1219,7 +1230,11 @@ int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(ret))
return ret;
return io_write(req, issue_flags);
ret = io_write(req, issue_flags);
if (ret != -EIOCBQUEUED)
io_detach_dmabuf_map(req);
return ret;
}
void io_rw_fail(struct io_kiocb *req)

29
lib/iov_iter.c
View File

@ -575,7 +575,8 @@ void iov_iter_advance(struct iov_iter *i, size_t size)
{
if (unlikely(i->count < size))
size = i->count;
if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i)) ||
unlikely(iov_iter_is_dmabuf_map(i))) {
i->iov_offset += size;
i->count -= size;
} else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {
@ -631,7 +632,8 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll)
return;
}
unroll -= i->iov_offset;
if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) {
if (iov_iter_is_xarray(i) || iter_is_ubuf(i) ||
iov_iter_is_dmabuf_map(i)) {
BUG(); /* We should never go beyond the start of the specified
* range since we might then be straying into pages that
* aren't pinned.
@ -775,6 +777,20 @@ void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
}
EXPORT_SYMBOL(iov_iter_xarray);
void iov_iter_dmabuf_map(struct iov_iter *i, unsigned int direction,
struct dma_buf_io_map *map,
loff_t off, size_t count)
{
WARN_ON(direction & ~(READ | WRITE));
*i = (struct iov_iter){
.iter_type = ITER_DMABUF_MAP,
.data_source = direction,
.dmabuf_map = map,
.count = count,
.iov_offset = off,
};
}
/**
* iov_iter_discard - Initialise an I/O iterator that discards data
* @i: The iterator to initialise.
@ -856,7 +872,7 @@ unsigned long iov_iter_alignment(const struct iov_iter *i)
return iov_iter_alignment_bvec(i);
/* With both xarray and folioq types, we're dealing with whole folios. */
if (iov_iter_is_folioq(i))
if (iov_iter_is_folioq(i) || iov_iter_is_dmabuf_map(i))
return i->iov_offset | i->count;
if (iov_iter_is_xarray(i))
return (i->xarray_start + i->iov_offset) | i->count;
@ -872,7 +888,7 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
size_t size = i->count;
unsigned k;
if (iter_is_ubuf(i))
if (iter_is_ubuf(i) || iov_iter_is_dmabuf_map(i))
return 0;
if (WARN_ON(!iter_is_iovec(i)))
@ -1469,11 +1485,12 @@ EXPORT_SYMBOL_GPL(import_ubuf);
void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
{
if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) &&
!iter_is_ubuf(i)) && !iov_iter_is_kvec(i))
!iter_is_ubuf(i) && !iov_iter_is_kvec(i) &&
!iov_iter_is_dmabuf_map(i)))
return;
i->iov_offset = state->iov_offset;
i->count = state->count;
if (iter_is_ubuf(i))
if (iter_is_ubuf(i) || iov_iter_is_dmabuf_map(i))
return;
/*
* For the *vec iters, nr_segs + iov is constant - if we increment