From 7607c44c157d343223510c8ffdf7206fdd2a6213 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Tue, 9 Nov 2021 19:47:22 +0900 Subject: [PATCH 01/10] block: Hold invalidate_lock in BLKDISCARD ioctl When BLKDISCARD ioctl and data read race, the data read leaves stale page cache. To avoid the stale page cache, hold invalidate_lock of the block device file mapping. The stale page cache is observed when blktests test case block/009 is repeated hundreds of times. This patch can be applied back to the stable kernel version v5.15.y with slight patch edit. Rework is required for older stable kernels. Fixes: 351499a172c0 ("block: Invalidate cache on discard v2") Signed-off-by: Shin'ichiro Kawasaki Cc: stable@vger.kernel.org # v5.15 Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211109104723.835533-2-shinichiro.kawasaki@wdc.com Signed-off-by: Jens Axboe --- block/ioctl.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/block/ioctl.c b/block/ioctl.c index d6af0ac97e57..9fa87f64f703 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -113,6 +113,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, uint64_t range[2]; uint64_t start, len; struct request_queue *q = bdev_get_queue(bdev); + struct inode *inode = bdev->bd_inode; int err; if (!(mode & FMODE_WRITE)) @@ -135,12 +136,17 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, if (start + len > bdev_nr_bytes(bdev)) return -EINVAL; + filemap_invalidate_lock(inode->i_mapping); err = truncate_bdev_range(bdev, mode, start, start + len - 1); if (err) - return err; + goto fail; - return blkdev_issue_discard(bdev, start >> 9, len >> 9, - GFP_KERNEL, flags); + err = blkdev_issue_discard(bdev, start >> 9, len >> 9, + GFP_KERNEL, flags); + +fail: + filemap_invalidate_unlock(inode->i_mapping); + return err; } static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, From 35e4c6c1a2fc2eb11b9306e95cda1fa06a511948 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Tue, 9 Nov 2021 19:47:23 +0900 Subject: [PATCH 02/10] block: Hold invalidate_lock in BLKZEROOUT ioctl When BLKZEROOUT ioctl and data read race, the data read leaves stale page cache. To avoid the stale page cache, hold invalidate_lock of the block device file mapping. The stale page cache is observed when blktests test case block/009 is modified to call "blkdiscard -z" command and repeated hundreds of times. This patch can be applied back to the stable kernel version v5.15.y. Rework is required for older stable kernels. Fixes: 22dd6d356628 ("block: invalidate the page cache when issuing BLKZEROOUT") Signed-off-by: Shin'ichiro Kawasaki Cc: stable@vger.kernel.org # v5.15 Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211109104723.835533-3-shinichiro.kawasaki@wdc.com Signed-off-by: Jens Axboe --- block/ioctl.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/block/ioctl.c b/block/ioctl.c index 9fa87f64f703..0a1d10ac2e1a 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -154,6 +154,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, { uint64_t range[2]; uint64_t start, end, len; + struct inode *inode = bdev->bd_inode; int err; if (!(mode & FMODE_WRITE)) @@ -176,12 +177,17 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, return -EINVAL; /* Invalidate the page cache, including dirty pages */ + filemap_invalidate_lock(inode->i_mapping); err = truncate_bdev_range(bdev, mode, start, end); if (err) - return err; + goto fail; - return blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL, - BLKDEV_ZERO_NOUNMAP); + err = blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL, + BLKDEV_ZERO_NOUNMAP); + +fail: + filemap_invalidate_unlock(inode->i_mapping); + return err; } static int put_ushort(unsigned short __user *argp, unsigned short val) From ecaf97f474447821ade290cebbe82bc9b6b23cff Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Nov 2021 15:08:11 -0700 Subject: [PATCH 03/10] block: use enum type for blk_mq_alloc_data->rq_flags kernel test robot reports that we now trigger some sparse warnings: block/blk-mq.h:169:32: sparse: sparse: restricted req_flags_t degrades to integer block/blk-mq.h:169:32: sparse: sparse: restricted req_flags_t degrades to integer block/blk-mq.h:169:32: sparse: sparse: restricted req_flags_t degrades to integer which is due to ->rq_flags being an unsigned int, rather than the stronger type req_flags_t enum. Change the type to req_flags_t to silence this warning. Fixes: 56f8da642bd8 ("block: add rq_flags to struct blk_mq_alloc_data") Reported-by: kernel test robot Reviewed-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/blk-mq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/blk-mq.h b/block/blk-mq.h index cb0b5482ca5e..39370bbdf3b6 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -149,7 +149,7 @@ struct blk_mq_alloc_data { blk_mq_req_flags_t flags; unsigned int shallow_depth; unsigned int cmd_flags; - unsigned int rq_flags; + req_flags_t rq_flags; /* allocate multiple requests/tags in one go */ unsigned int nr_tags; From 278167fd2f8ffe679351605fe03e29ff3ab8db18 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Tue, 9 Nov 2021 16:29:49 -0800 Subject: [PATCH 04/10] block: add __must_check for *add_disk*() callers Now that we have done a spring cleaning on all drivers and added error checking / handling, let's keep it that way and ensure no new drivers fail to stick with it. Reviewed-by: Christoph Hellwig Signed-off-by: Luis Chamberlain Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20211110002949.999380-1-mcgrof@kernel.org Signed-off-by: Jens Axboe --- block/genhd.c | 6 +++--- include/linux/genhd.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/block/genhd.c b/block/genhd.c index ca2fbab1d425..c5392cc24d37 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -394,8 +394,8 @@ static void disk_scan_partitions(struct gendisk *disk) * This function registers the partitioning information in @disk * with the kernel. */ -int device_add_disk(struct device *parent, struct gendisk *disk, - const struct attribute_group **groups) +int __must_check device_add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups) { struct device *ddev = disk_to_dev(disk); @@ -544,7 +544,7 @@ int device_add_disk(struct device *parent, struct gendisk *disk, out_free_ext_minor: if (disk->major == BLOCK_EXT_MAJOR) blk_free_ext_minor(disk->first_minor); - return WARN_ON_ONCE(ret); /* keep until all callers handle errors */ + return ret; } EXPORT_SYMBOL(device_add_disk); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 462634b4b48f..74c410263113 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -205,9 +205,9 @@ static inline dev_t disk_devt(struct gendisk *disk) void disk_uevent(struct gendisk *disk, enum kobject_action action); /* block/genhd.c */ -int device_add_disk(struct device *parent, struct gendisk *disk, - const struct attribute_group **groups); -static inline int add_disk(struct gendisk *disk) +int __must_check device_add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups); +static inline int __must_check add_disk(struct gendisk *disk) { return device_add_disk(NULL, disk, NULL); } From 438cd74223c0029cd7409ca99aaf92e0972f3557 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Nov 2021 17:32:32 -0700 Subject: [PATCH 05/10] block: fix kerneldoc for disk_register_independent_access__ranges() The naming got changed as part of a revision of the patchset, but the kerneldoc apparently never got updated. Fix it. Reported-by: kernel test robot Fixes: a2247f19ee1c ("block: Add independent access ranges support") Signed-off-by: Jens Axboe --- block/blk-ia-ranges.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/blk-ia-ranges.c b/block/blk-ia-ranges.c index c246c425d0d7..b925f3db3ab7 100644 --- a/block/blk-ia-ranges.c +++ b/block/blk-ia-ranges.c @@ -104,8 +104,8 @@ static struct kobj_type blk_ia_ranges_ktype = { }; /** - * disk_register_ia_ranges - register with sysfs a set of independent - * access ranges + * disk_register_independent_access_ranges - register with sysfs a set of + * independent access ranges * @disk: Target disk * @new_iars: New set of independent access ranges * From 10f7335e3627b4efa341ef8ac457f2c0770c5c19 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 11 Nov 2021 16:51:33 +0800 Subject: [PATCH 06/10] blk-mq: don't grab ->q_usage_counter in blk_mq_sched_bio_merge blk_mq_sched_bio_merge is only called from blk-mq.c:blk_attempt_bio_merge(), which is called when queue usage counter is grabbed already: 1) blk_mq_get_new_requests() 2) blk_mq_get_request() - cached request in current plug owns one queue usage counter So don't grab ->q_usage_counter in blk_mq_sched_bio_merge(), and more importantly this nest way causes hang in blk_mq_freeze_queue_wait(). Cc: Christoph Hellwig Signed-off-by: Ming Lei Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211111085134.345235-2-ming.lei@redhat.com Signed-off-by: Jens Axboe --- block/blk-mq-sched.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index 4be652fa38e7..ba21449439cc 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -370,9 +370,6 @@ bool blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, bool ret = false; enum hctx_type type; - if (bio_queue_enter(bio)) - return false; - if (e && e->type->ops.bio_merge) { ret = e->type->ops.bio_merge(q, bio, nr_segs); goto out_put; @@ -397,7 +394,6 @@ bool blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, spin_unlock(&ctx->lock); out_put: - blk_queue_exit(q); return ret; } From b131f2011115f3c18a49e17762486501496fea3c Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 11 Nov 2021 16:51:34 +0800 Subject: [PATCH 07/10] blk-mq: rename blk_attempt_bio_merge It is very annoying to have two block layer functions which share same name, so rename blk_attempt_bio_merge in blk-mq.c as blk_mq_attempt_bio_merge. Cc: Christoph Hellwig Signed-off-by: Ming Lei Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211111085134.345235-3-ming.lei@redhat.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 629cf421417f..f511db395c7f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2495,8 +2495,9 @@ static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug) return BLK_MAX_REQUEST_COUNT; } -static bool blk_attempt_bio_merge(struct request_queue *q, struct bio *bio, - unsigned int nr_segs, bool *same_queue_rq) +static bool blk_mq_attempt_bio_merge(struct request_queue *q, + struct bio *bio, unsigned int nr_segs, + bool *same_queue_rq) { if (!blk_queue_nomerges(q) && bio_mergeable(bio)) { if (blk_attempt_plug_merge(q, bio, nr_segs, same_queue_rq)) @@ -2524,7 +2525,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, return NULL; if (unlikely(!submit_bio_checks(bio))) goto put_exit; - if (blk_attempt_bio_merge(q, bio, nsegs, same_queue_rq)) + if (blk_mq_attempt_bio_merge(q, bio, nsegs, same_queue_rq)) goto put_exit; rq_qos_throttle(q, bio); @@ -2560,7 +2561,8 @@ static inline struct request *blk_mq_get_request(struct request_queue *q, if (rq && rq->q == q) { if (unlikely(!submit_bio_checks(bio))) return NULL; - if (blk_attempt_bio_merge(q, bio, nsegs, same_queue_rq)) + if (blk_mq_attempt_bio_merge(q, bio, nsegs, + same_queue_rq)) return NULL; plug->cached_rq = rq_list_next(rq); INIT_LIST_HEAD(&rq->queuelist); From 86399ea071099ec8ee0a83ac9ad67f7df96a50ad Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Thu, 11 Nov 2021 17:52:38 +0900 Subject: [PATCH 08/10] block: Hold invalidate_lock in BLKRESETZONE ioctl When BLKRESETZONE ioctl and data read race, the data read leaves stale page cache. The commit e5113505904e ("block: Discard page cache of zone reset target range") added page cache truncation to avoid stale page cache after the ioctl. However, the stale page cache still can be read during the reset zone operation for the ioctl. To avoid the stale page cache completely, hold invalidate_lock of the block device file mapping. Fixes: e5113505904e ("block: Discard page cache of zone reset target range") Signed-off-by: Shin'ichiro Kawasaki Cc: stable@vger.kernel.org # v5.15 Reviewed-by: Jan Kara Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20211111085238.942492-1-shinichiro.kawasaki@wdc.com Signed-off-by: Jens Axboe --- block/blk-zoned.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 1d0c76c18fc5..774ecc598bee 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -429,9 +429,10 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, op = REQ_OP_ZONE_RESET; /* Invalidate the page cache, including dirty pages. */ + filemap_invalidate_lock(bdev->bd_inode->i_mapping); ret = blkdev_truncate_zone_range(bdev, mode, &zrange); if (ret) - return ret; + goto fail; break; case BLKOPENZONE: op = REQ_OP_ZONE_OPEN; @@ -449,15 +450,9 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, ret = blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors, GFP_KERNEL); - /* - * Invalidate the page cache again for zone reset: writes can only be - * direct for zoned devices so concurrent writes would not add any page - * to the page cache after/during reset. The page cache may be filled - * again due to concurrent reads though and dropping the pages for - * these is fine. - */ - if (!ret && cmd == BLKRESETZONE) - ret = blkdev_truncate_zone_range(bdev, mode, &zrange); +fail: + if (cmd == BLKRESETZONE) + filemap_invalidate_unlock(bdev->bd_inode->i_mapping); return ret; } From b781d8db580c058ecd54ed7d5dde7f8270b25f5b Mon Sep 17 00:00:00 2001 From: Laibin Qiu Date: Fri, 12 Nov 2021 17:33:54 +0800 Subject: [PATCH 09/10] blkcg: Remove extra blkcg_bio_issue_init KASAN reports a use-after-free report when doing block test: ================================================================== [10050.967049] BUG: KASAN: use-after-free in submit_bio_checks+0x1539/0x1550 [10050.977638] Call Trace: [10050.978190] dump_stack+0x9b/0xce [10050.979674] print_address_description.constprop.6+0x3e/0x60 [10050.983510] kasan_report.cold.9+0x22/0x3a [10050.986089] submit_bio_checks+0x1539/0x1550 [10050.989576] submit_bio_noacct+0x83/0xc80 [10050.993714] submit_bio+0xa7/0x330 [10050.994435] mpage_readahead+0x380/0x500 [10050.998009] read_pages+0x1c1/0xbf0 [10051.002057] page_cache_ra_unbounded+0x4c2/0x6f0 [10051.007413] do_page_cache_ra+0xda/0x110 [10051.008207] force_page_cache_ra+0x23d/0x3d0 [10051.009087] page_cache_sync_ra+0xca/0x300 [10051.009970] generic_file_buffered_read+0xbea/0x2130 [10051.012685] generic_file_read_iter+0x315/0x490 [10051.014472] blkdev_read_iter+0x113/0x1b0 [10051.015300] aio_read+0x2ad/0x450 [10051.023786] io_submit_one+0xc8e/0x1d60 [10051.029855] __se_sys_io_submit+0x125/0x350 [10051.033442] do_syscall_64+0x2d/0x40 [10051.034156] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [10051.048733] Allocated by task 18598: [10051.049482] kasan_save_stack+0x19/0x40 [10051.050263] __kasan_kmalloc.constprop.1+0xc1/0xd0 [10051.051230] kmem_cache_alloc+0x146/0x440 [10051.052060] mempool_alloc+0x125/0x2f0 [10051.052818] bio_alloc_bioset+0x353/0x590 [10051.053658] mpage_alloc+0x3b/0x240 [10051.054382] do_mpage_readpage+0xddf/0x1ef0 [10051.055250] mpage_readahead+0x264/0x500 [10051.056060] read_pages+0x1c1/0xbf0 [10051.056758] page_cache_ra_unbounded+0x4c2/0x6f0 [10051.057702] do_page_cache_ra+0xda/0x110 [10051.058511] force_page_cache_ra+0x23d/0x3d0 [10051.059373] page_cache_sync_ra+0xca/0x300 [10051.060198] generic_file_buffered_read+0xbea/0x2130 [10051.061195] generic_file_read_iter+0x315/0x490 [10051.062189] blkdev_read_iter+0x113/0x1b0 [10051.063015] aio_read+0x2ad/0x450 [10051.063686] io_submit_one+0xc8e/0x1d60 [10051.064467] __se_sys_io_submit+0x125/0x350 [10051.065318] do_syscall_64+0x2d/0x40 [10051.066082] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [10051.067455] Freed by task 13307: [10051.068136] kasan_save_stack+0x19/0x40 [10051.068931] kasan_set_track+0x1c/0x30 [10051.069726] kasan_set_free_info+0x1b/0x30 [10051.070621] __kasan_slab_free+0x111/0x160 [10051.071480] kmem_cache_free+0x94/0x460 [10051.072256] mempool_free+0xd6/0x320 [10051.072985] bio_free+0xe0/0x130 [10051.073630] bio_put+0xab/0xe0 [10051.074252] bio_endio+0x3a6/0x5d0 [10051.074984] blk_update_request+0x590/0x1370 [10051.075870] scsi_end_request+0x7d/0x400 [10051.076667] scsi_io_completion+0x1aa/0xe50 [10051.077503] scsi_softirq_done+0x11b/0x240 [10051.078344] blk_mq_complete_request+0xd4/0x120 [10051.079275] scsi_mq_done+0xf0/0x200 [10051.080036] virtscsi_vq_done+0xbc/0x150 [10051.080850] vring_interrupt+0x179/0x390 [10051.081650] __handle_irq_event_percpu+0xf7/0x490 [10051.082626] handle_irq_event_percpu+0x7b/0x160 [10051.083527] handle_irq_event+0xcc/0x170 [10051.084297] handle_edge_irq+0x215/0xb20 [10051.085122] asm_call_irq_on_stack+0xf/0x20 [10051.085986] common_interrupt+0xae/0x120 [10051.086830] asm_common_interrupt+0x1e/0x40 ================================================================== Bio will be checked at beginning of submit_bio_noacct(). If bio needs to be throttled, it will start the timer and stop submit bio directly. Bio will submit in blk_throtl_dispatch_work_fn() when the timer expires. But in the current process, if bio is throttled, it will still set bio issue->value by blkcg_bio_issue_init(). This is redundant and may cause the above use-after-free. CPU0 CPU1 submit_bio submit_bio_noacct submit_bio_checks blk_throtl_bio() <=mod_timer(&sq->pending_timer blk_throtl_dispatch_work_fn submit_bio_noacct() <= bio have throttle tag, will throw directly and bio issue->value will be set here bio_endio() bio_put() bio_free() <= free this bio blkcg_bio_issue_init(bio) <= bio has been freed and will lead to UAF return BLK_QC_T_NONE Fix this by remove extra blkcg_bio_issue_init. Fixes: e439bedf6b24 (blkcg: consolidate bio_issue_init() to be a part of core) Signed-off-by: Laibin Qiu Link: https://lore.kernel.org/r/20211112093354.3581504-1-qiulaibin@huawei.com Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-core.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index b043de2baaac..9ee32f85d74e 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -809,10 +809,8 @@ noinline_for_stack bool submit_bio_checks(struct bio *bio) if (unlikely(!current->io_context)) create_task_io_context(current, GFP_ATOMIC, q->node); - if (blk_throtl_bio(bio)) { - blkcg_bio_issue_init(bio); + if (blk_throtl_bio(bio)) return false; - } blk_cgroup_bio_start(bio); blkcg_bio_issue_init(bio); From b637108a4022951dcc71b672bd101ebe24ad26d5 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 12 Nov 2021 20:47:15 +0800 Subject: [PATCH 10/10] blk-mq: fix filesystem I/O request allocation submit_bio_checks() may update bio->bi_opf, so we have to initialize blk_mq_alloc_data.cmd_flags with bio->bi_opf after submit_bio_checks() returns when allocating new request. In case of using cached request, fallback to allocate new request if cached rq isn't compatible with the incoming bio, otherwise change rq->cmd_flags with incoming bio->bi_opf. Fixes: 900e080752025f00 ("block: move queue enter logic into blk_mq_submit_bio()") Reported-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Cc: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-mq.c | 39 ++++++++++++++++++++++++++++++--------- block/blk-mq.h | 26 +++++++++++++++----------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index f511db395c7f..3ab34c4f20da 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2521,12 +2521,8 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, }; struct request *rq; - if (unlikely(bio_queue_enter(bio))) - return NULL; - if (unlikely(!submit_bio_checks(bio))) - goto put_exit; if (blk_mq_attempt_bio_merge(q, bio, nsegs, same_queue_rq)) - goto put_exit; + return NULL; rq_qos_throttle(q, bio); @@ -2543,19 +2539,32 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, rq_qos_cleanup(q, bio); if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); -put_exit: - blk_queue_exit(q); + return NULL; } +static inline bool blk_mq_can_use_cached_rq(struct request *rq, + struct bio *bio) +{ + if (blk_mq_get_hctx_type(bio->bi_opf) != rq->mq_hctx->type) + return false; + + if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf)) + return false; + + return true; +} + static inline struct request *blk_mq_get_request(struct request_queue *q, struct blk_plug *plug, struct bio *bio, unsigned int nsegs, bool *same_queue_rq) { + struct request *rq; + bool checked = false; + if (plug) { - struct request *rq; rq = rq_list_peek(&plug->cached_rq); if (rq && rq->q == q) { @@ -2564,6 +2573,10 @@ static inline struct request *blk_mq_get_request(struct request_queue *q, if (blk_mq_attempt_bio_merge(q, bio, nsegs, same_queue_rq)) return NULL; + checked = true; + if (!blk_mq_can_use_cached_rq(rq, bio)) + goto fallback; + rq->cmd_flags = bio->bi_opf; plug->cached_rq = rq_list_next(rq); INIT_LIST_HEAD(&rq->queuelist); rq_qos_throttle(q, bio); @@ -2571,7 +2584,15 @@ static inline struct request *blk_mq_get_request(struct request_queue *q, } } - return blk_mq_get_new_requests(q, plug, bio, nsegs, same_queue_rq); +fallback: + if (unlikely(bio_queue_enter(bio))) + return NULL; + if (!checked && !submit_bio_checks(bio)) + return NULL; + rq = blk_mq_get_new_requests(q, plug, bio, nsegs, same_queue_rq); + if (!rq) + blk_queue_exit(q); + return rq; } /** diff --git a/block/blk-mq.h b/block/blk-mq.h index 39370bbdf3b6..8acfa650f575 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -89,15 +89,7 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue * return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; } -/* - * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue - * @q: request queue - * @flags: request command flags - * @ctx: software queue cpu ctx - */ -static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, - unsigned int flags, - struct blk_mq_ctx *ctx) +static inline enum hctx_type blk_mq_get_hctx_type(unsigned int flags) { enum hctx_type type = HCTX_TYPE_DEFAULT; @@ -108,8 +100,20 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, type = HCTX_TYPE_POLL; else if ((flags & REQ_OP_MASK) == REQ_OP_READ) type = HCTX_TYPE_READ; - - return ctx->hctxs[type]; + return type; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @ctx: software queue cpu ctx + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + struct blk_mq_ctx *ctx) +{ + return ctx->hctxs[blk_mq_get_hctx_type(flags)]; } /*