mirror of
https://github.com/torvalds/linux.git
synced 2026-07-28 10:09:10 +02:00
e7c1627afd
bio_integrity_alloc_buf usage of GFP_ flags is messed up. For one it
mixes GFP_NOFS and GFP_NOIO for neighbouring allocations, but it also
makes the allocations fail more often than needed. That code was copied
from bio_alloc_bioset which needs to do that so that it can punt to the
rescuer workqueue, but none of that is needed for the integrity
allocations that either sits in the file system or at the very bottom
of the I/O stack. Failing early means we'll do a fully waiting
allocation from the mempool ->alloc callback which is usually much
larger than required.
Fix this by passing a gfp_t so that the file system path can pass
GFP_NOFS and the auto-integrity code can pass GFP_NOIO, and don't
modify the allocation type except for disabling warnings.
Fixes: ec7f31b2a2 ("block: make bio auto-integrity deadlock safe")
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Link: https://patch.msgid.link/20260624080014.1998650-2-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
134 lines
3.9 KiB
C
134 lines
3.9 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2007, 2008, 2009 Oracle Corporation
|
|
* Written by: Martin K. Petersen <martin.petersen@oracle.com>
|
|
*
|
|
* Automatically generate and verify integrity data on PI capable devices if the
|
|
* bio submitter didn't provide PI itself. This ensures that kernel verifies
|
|
* data integrity even if the file system (or other user of the block device) is
|
|
* not aware of PI.
|
|
*/
|
|
#include <linux/blk-integrity.h>
|
|
#include <linux/t10-pi.h>
|
|
#include <linux/workqueue.h>
|
|
#include "blk.h"
|
|
|
|
struct bio_integrity_data {
|
|
struct bio *bio;
|
|
struct bvec_iter saved_bio_iter;
|
|
struct work_struct work;
|
|
struct bio_integrity_payload bip;
|
|
struct bio_vec bvec;
|
|
};
|
|
|
|
static struct kmem_cache *bid_slab;
|
|
static mempool_t bid_pool;
|
|
static struct workqueue_struct *kintegrityd_wq;
|
|
|
|
static void bio_integrity_finish(struct bio_integrity_data *bid)
|
|
{
|
|
bid->bio->bi_integrity = NULL;
|
|
bid->bio->bi_opf &= ~REQ_INTEGRITY;
|
|
bio_integrity_free_buf(&bid->bip);
|
|
mempool_free(bid, &bid_pool);
|
|
}
|
|
|
|
static void bio_integrity_verify_fn(struct work_struct *work)
|
|
{
|
|
struct bio_integrity_data *bid =
|
|
container_of(work, struct bio_integrity_data, work);
|
|
struct bio *bio = bid->bio;
|
|
|
|
bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter);
|
|
bio_integrity_finish(bid);
|
|
bio_endio(bio);
|
|
}
|
|
|
|
#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
|
|
static bool bip_should_check(struct bio_integrity_payload *bip)
|
|
{
|
|
return bip->bip_flags & BIP_CHECK_FLAGS;
|
|
}
|
|
|
|
/**
|
|
* __bio_integrity_endio - Integrity I/O completion function
|
|
* @bio: Protected bio
|
|
*
|
|
* Normally I/O completion is done in interrupt context. However, verifying I/O
|
|
* integrity is a time-consuming task which must be run in process context.
|
|
*
|
|
* This function postpones completion accordingly.
|
|
*/
|
|
bool __bio_integrity_endio(struct bio *bio)
|
|
{
|
|
struct bio_integrity_payload *bip = bio_integrity(bio);
|
|
struct bio_integrity_data *bid =
|
|
container_of(bip, struct bio_integrity_data, bip);
|
|
|
|
if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
|
|
bip_should_check(bip)) {
|
|
INIT_WORK(&bid->work, bio_integrity_verify_fn);
|
|
queue_work(kintegrityd_wq, &bid->work);
|
|
return false;
|
|
}
|
|
|
|
bio_integrity_finish(bid);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* bio_integrity_prep - Prepare bio for integrity I/O
|
|
* @bio: bio to prepare
|
|
* @action: preparation action needed (BI_ACT_*)
|
|
*
|
|
* Allocate the integrity payload. For writes, generate the integrity metadata
|
|
* and for reads, setup the completion handler to verify the metadata.
|
|
*
|
|
* This is used for bios that do not have user integrity payloads attached.
|
|
*/
|
|
void bio_integrity_prep(struct bio *bio, unsigned int action)
|
|
{
|
|
struct bio_integrity_data *bid;
|
|
|
|
bid = mempool_alloc(&bid_pool, GFP_NOIO);
|
|
bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
|
|
bid->bio = bio;
|
|
bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
|
|
bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO);
|
|
if (action & BI_ACT_CHECK)
|
|
bio_integrity_setup_default(bio);
|
|
|
|
/* Auto-generate integrity metadata if this is a write */
|
|
if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip))
|
|
bio_integrity_generate(bio);
|
|
else
|
|
bid->saved_bio_iter = bio->bi_iter;
|
|
}
|
|
EXPORT_SYMBOL(bio_integrity_prep);
|
|
|
|
void blk_flush_integrity(void)
|
|
{
|
|
flush_workqueue(kintegrityd_wq);
|
|
}
|
|
|
|
static int __init blk_integrity_auto_init(void)
|
|
{
|
|
bid_slab = kmem_cache_create("bio_integrity_data",
|
|
sizeof(struct bio_integrity_data), 0,
|
|
SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
|
|
|
|
if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
|
|
panic("bio: can't create integrity pool\n");
|
|
|
|
/*
|
|
* kintegrityd won't block much but may burn a lot of CPU cycles.
|
|
* Make it highpri CPU intensive wq with max concurrency of 1.
|
|
*/
|
|
kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
|
|
WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_PERCPU, 1);
|
|
if (!kintegrityd_wq)
|
|
panic("Failed to create kintegrityd\n");
|
|
return 0;
|
|
}
|
|
subsys_initcall(blk_integrity_auto_init);
|