diff --git a/block/bio.c b/block/bio.c index 57ee335899f3..c89a0d5d6e61 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1285,6 +1285,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned mem_align_mask, unsigned len_align_mask) { iov_iter_extraction_t flags = 0; + int ret; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; @@ -1304,34 +1305,47 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, flags |= ITER_ALLOW_P2PDMA; do { - ssize_t ret; + ssize_t size; - ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, + size = iov_iter_extract_bvecs(iter, bio->bi_io_vec, BIO_MAX_SIZE - bio->bi_iter.bi_size, &bio->bi_vcnt, bio->bi_max_vecs, mem_align_mask, flags); - if (ret <= 0) { - /* - * A misaligned vector fails the whole I/O. Release any - * pages pinned by earlier iterations before returning - * since this bio won't be submitted to release them. - */ - if (ret == -EINVAL) { - bio_release_pages(bio, false); - bio_clear_flag(bio, BIO_PAGE_PINNED); - bio->bi_vcnt = 0; - } + if (size <= 0) { + /* A misaligned vector fails the whole I/O */ + if (size == -EINVAL) + goto out_release_pages; if (!bio->bi_vcnt) - return ret; + return size; break; } - bio->bi_iter.bi_size += ret; + bio->bi_iter.bi_size += size; } while (iov_iter_count(iter) && !bio_full(bio, 0)); if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) bio->bi_opf |= REQ_NOMERGE; - return bio_iov_iter_align_down(bio, iter, + ret = bio_iov_iter_align_down(bio, iter, &bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask); + if (ret) + return ret; + + /* + * An atomic write is submitted as a single bio, so it has to cover + * the whole iterator or it would be torn. + */ + if ((bio->bi_opf & REQ_ATOMIC) && iov_iter_count(iter)) + goto out_release_pages; + return 0; + +out_release_pages: + /* + * Release the pages pinned so far before failing, since this bio won't + * be submitted to release them. + */ + bio_release_pages(bio, false); + bio_clear_flag(bio, BIO_PAGE_PINNED); + bio->bi_vcnt = 0; + return -EINVAL; } static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size, diff --git a/block/fops.c b/block/fops.c index 2022c7cb2ee3..1ac3fa285090 100644 --- a/block/fops.c +++ b/block/fops.c @@ -357,6 +357,12 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_ioprio = iocb->ki_ioprio; + if (iocb->ki_flags & IOCB_ATOMIC) + bio->bi_opf |= REQ_ATOMIC; + + if (iocb->ki_flags & IOCB_NOWAIT) + bio->bi_opf |= REQ_NOWAIT; + /* * Users don't rely on the iterator being in any particular * state for async I/O returning -EIOCBQUEUED, hence we can @@ -386,12 +392,6 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, goto out_bio_put; } - if (iocb->ki_flags & IOCB_ATOMIC) - bio->bi_opf |= REQ_ATOMIC; - - if (iocb->ki_flags & IOCB_NOWAIT) - bio->bi_opf |= REQ_NOWAIT; - if (iocb->ki_flags & IOCB_HIPRI) { bio->bi_opf |= REQ_POLLED; submit_bio(bio);