- 6 minor cleanups found by Claude Opus 4.6

- small cleanups in dm core, dm-cache, dm-switch, dm-inlinecrypt, dm-vdo
 
 - improve validation of metadata in dm-pcache
 
 - dm: fix resume-vs-remove ioctl race condition
 
 - dm: fix race condition when issuing table load ioctls concurrently
 
 - fix dm-raid1 and dm-io, so that they work with unaligned bio vectors
 
 - dm-integrity: use keyed markers as discard fillers
 
 - improve metadata validation in dm-array
 
 - fix dm-stats crash on memory allocation failure
 
 - fix dm-dust, so that it works if it is not the first target in a table
 
 - dm-era: fix superblock refcount leak on snapshot failure
 -----BEGIN PGP SIGNATURE-----
 
 iIoEABYIADIWIQRnH8MwLyZDhyYfesYTAyx9YGnhbQUCaoNttBQcbXBhdG9ja2FA
 cmVkaGF0LmNvbQAKCRATAyx9YGnhbThWAQCYDN/SclnydRYV7IKzV/yyd75esUOi
 lWC3kMbiykSD0AEAmdQeMEqwxpX8i1njnPrcm67i4+LEKMK4X+ApLwV30gQ=
 =SOAs
 -----END PGP SIGNATURE-----

Merge tag 'for-7.3/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mikulas Patocka:

 - minor cleanups found by Claude Opus 4.6

 - small cleanups in dm core, dm-cache, dm-switch, dm-inlinecrypt,
   dm-vdo

 - improve validation of metadata in dm-pcache

 - fix resume-vs-remove ioctl race condition

 - fix race condition when issuing table load ioctls concurrently

 - fix dm-raid1 and dm-io, so that they work with unaligned bio vectors

 - dm-integrity: use keyed markers as discard fillers

 - improve metadata validation in dm-array

 - fix dm-stats crash on memory allocation failure

 - fix dm-dust, so that it works if it is not the first target in a
   table

 - dm-era: fix superblock refcount leak on snapshot failure

* tag 'for-7.3/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: (46 commits)
  dm-era: fix shadowed superblock leak on take-snap failure
  dm dust: make badblock messages target-relative
  dm-stats: fix a crash if allocation of per-cpu data fails
  dm array: reject an array block whose value size is not the caller's
  dm array: validate array block headers on read
  dm-integrity: replace forgeable discard filler with a keyed sector marker
  dm vdo indexer: embed geometry in parent structures
  dm vdo indexer: simplify sub-index parameter calculations
  dm-pcache: remove unused 'cache' parameter from cache_key_gc()
  docs: device-mapper: dm-inlinecrypt: fix 'bellow' spelling
  dm-pcache: remove unused miss_read_end_work_fn declaration
  dm-io: report non-retryable errors separatedly
  dm-io: clone the source bio instead of copying its biovec
  dm: fix race when loading and unloading a table
  dm: fix resume-vs-remove race
  dm-pcache: remove unused 'allocated' variable in cache_data_alloc()
  dm-pcache: replace tabs with spaces in comments to fix ASCII diagram alignment
  dm-pcache: fix use-after-free and invalid seg operations in kset_replay()
  dm-pcache: fix implicit u8 truncation of gc_percent in message handler
  dm raid1: reserve space for NUL-terminator in build_constructor_string()
  ...

Documentation/admin-guide/device-mapper/dm-ima.rst

@@ -424,7 +424,8 @@ section above) has the following data format for 'integrity' target.
 
  target_attributes := <target_name> "," <target_version> "," <dev_name> "," <start>
                       <tag_size> "," <mode> "," [<meta_device> ","] [<block_size> ","] <recalculate> ","
-                      <allow_discards> "," <fix_padding> "," <fix_hmac> "," <legacy_recalculate> ","
+                      <allow_discards> "," <allow_discards_keyed> "," <fix_padding> "," <fix_hmac> ","
+                      <legacy_recalculate> ","
                       <journal_sectors> "," <interleave_sectors> "," <buffer_sectors> ";"
 
  target_name := "target_name=integrity"
@@ -438,6 +439,7 @@ section above) has the following data format for 'integrity' target.
  block_size := "block_size=" <N>
  recalculate := "recalculate=" <yes_no>
  allow_discards := "allow_discards=" <yes_no>
+ allow_discards_keyed := "allow_discards_keyed=" <yes_no>
  fix_padding := "fix_padding=" <yes_no>
  fix_hmac := "fix_hmac=" <yes_no>
  legacy_recalculate := "legacy_recalculate=" <yes_no>
@@ -455,7 +457,8 @@ section above) has the following data format for 'integrity' target.
  dm_version=4.45.0;
  name=integrity1,uuid=,major=253,minor=1,minor_count=1,num_targets=1;
  target_index=0,target_begin=0,target_len=7856,target_name=integrity,target_version=1.10.0,
- dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n,fix_padding=n,
+ dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n,
+ allow_discards_keyed=n,fix_padding=n,
  fix_hmac=n,legacy_recalculate=n,journal_sectors=88,interleave_sectors=32768,buffer_sectors=128;
 
 

Documentation/admin-guide/device-mapper/dm-inlinecrypt.rst

@@ -33,7 +33,7 @@ Parameters::
     or it can be passed as <key_string> prefixed with single colon
     character (':') for keys residing in kernel keyring service.
     You can only use key sizes that are valid for the selected cipher.
-    Note that the size in bytes of a valid key must be in bellow range.
+    Note that the size in bytes of a valid key must be in below range.
 
         [BLK_CRYPTO_KEY_TYPE_RAW, BLK_CRYPTO_KEY_TYPE_HW_WRAPPED]
 

Documentation/admin-guide/device-mapper/dm-integrity.rst

@@ -190,6 +190,19 @@ allow_discards
 	Allow block discard requests (a.k.a. TRIM) for the integrity device.
 	Discards are only allowed to devices using internal hash.
 
+	A discarded block is marked with a constant filler tag that anyone
+	with raw write access to the backing device can forge without the
+	key. Use allow_discards_keyed instead on new volumes.
+
+allow_discards_keyed
+	Like allow_discards, but marks a discarded block with a keyed
+	checksum of the sector number, HMAC_key(salt || sector), instead of
+	the constant filler tag, so it can't be forged without the
+	integrity key.
+
+	Not compatible with volumes that already have discarded blocks
+	marked the old way; only use on a freshly formatted volume.
+
 fix_padding
 	Use a smaller padding of the tag area that is more
 	space-efficient. If this option is not present, large padding is

drivers/md/dm-bufio.c

@@ -1287,11 +1287,11 @@ static void free_buffer(struct dm_buffer *b)
  * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
  * that the request was handled directly with bio interface.
  */
-static void dmio_complete(unsigned long error, void *context)
+static void dmio_complete(unsigned long error, unsigned long unsup, void *context)
 {
 	struct dm_buffer *b = context;
 
-	b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
+	b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : unlikely(unsup != 0) ? BLK_STS_NOTSUPP : 0);
 }
 
 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
@@ -1319,7 +1319,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
 		io_req.mem.ptr.vma = (char *)b->data + offset;
 	}
 
-	r = dm_io(&io_req, 1, &region, NULL, ioprio);
+	r = dm_io(&io_req, 1, &region, NULL, NULL, ioprio);
 	if (unlikely(r))
 		b->end_io(b, errno_to_blk_status(r));
 }
@@ -2220,7 +2220,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
 	if (WARN_ON_ONCE(dm_bufio_in_request()))
 		return -EINVAL;
 
-	return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
+	return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
 
@@ -2246,7 +2246,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c
 	if (WARN_ON_ONCE(dm_bufio_in_request()))
 		return -EINVAL; /* discards are optional */
 
-	return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
+	return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
 

drivers/md/dm-cache-target.c

@@ -16,6 +16,7 @@
 #include <linux/dm-kcopyd.h>
 #include <linux/jiffies.h>
 #include <linux/init.h>
+#include <linux/kstrtox.h>
 #include <linux/mempool.h>
 #include <linux/module.h>
 #include <linux/rwsem.h>
@@ -3311,42 +3312,46 @@ struct cblock_range {
 	dm_cblock_t end;
 };
 
+static inline dm_cblock_t cblock_succ(dm_cblock_t b)
+{
+	return to_cblock(from_cblock(b) + 1);
+}
+
 /*
  * A cache block range can take two forms:
  *
  * i) A single cblock, eg. '3456'
  * ii) A begin and end cblock with a dash between, eg. 123-234
  */
-static int parse_cblock_range(struct cache *cache, const char *str,
+static int parse_cblock_range(struct cache *cache, char *str,
 			      struct cblock_range *result)
 {
-	char dummy;
-	uint64_t b, e;
+	char *blocknr = strsep(&str, "-");
+	unsigned int b, e;
 	int r;
 
-	/*
-	 * Try and parse form (ii) first.
-	 */
-	r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
+	r = kstrtouint(blocknr, 10, &b);
+	if (r)
+		goto bad;
+
+	result->begin = to_cblock(b);
+
+	if (str) {
+		blocknr = str;
+
+		r = kstrtouint(blocknr, 10, &e);
+		if (r)
+			goto bad;
 
-	if (r == 2) {
-		result->begin = to_cblock(b);
 		result->end = to_cblock(e);
-		return 0;
+	} else {
+		result->end = cblock_succ(result->begin);
 	}
 
-	/*
-	 * That didn't work, try form (i).
-	 */
-	r = sscanf(str, "%llu%c", &b, &dummy);
+	return 0;
 
-	if (r == 1) {
-		result->begin = to_cblock(b);
-		result->end = to_cblock(from_cblock(result->begin) + 1u);
-		return 0;
-	}
-
-	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
+bad:
+	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), blocknr);
 	return -EINVAL;
 }
 
@@ -3377,11 +3382,6 @@ static int validate_cblock_range(struct cache *cache, struct cblock_range *range
 	return 0;
 }
 
-static inline dm_cblock_t cblock_succ(dm_cblock_t b)
-{
-	return to_cblock(from_cblock(b) + 1);
-}
-
 static int request_invalidation(struct cache *cache, struct cblock_range *range)
 {
 	int r = 0;
@@ -3405,7 +3405,7 @@ static int request_invalidation(struct cache *cache, struct cblock_range *range)
 }
 
 static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count,
-					      const char **cblock_ranges)
+					      char **cblock_ranges)
 {
 	int r = 0;
 	unsigned int i;
@@ -3460,7 +3460,7 @@ static int cache_message(struct dm_target *ti, unsigned int argc, char **argv,
 	}
 
 	if (!strcasecmp(argv[0], "invalidate_cblocks"))
-		return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
+		return process_invalidate_cblocks_message(cache, argc - 1, argv + 1);
 
 	if (argc != 2)
 		return -EINVAL;

drivers/md/dm-dust.c

@@ -196,7 +196,6 @@ static int __dust_map_write(struct dust_device *dd, sector_t thisblock)
 		dd->badblock_count--;
 		kfree(bblk);
 		if (!dd->quiet_mode) {
-			sector_div(thisblock, dd->sect_per_block);
 			DMINFO("block %llu removed from badblocklist by write",
 			       (unsigned long long)thisblock);
 		}
@@ -224,15 +223,16 @@ static int dust_map_write(struct dust_device *dd, sector_t thisblock,
 static int dust_map(struct dm_target *ti, struct bio *bio)
 {
 	struct dust_device *dd = ti->private;
+	sector_t dust_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
 	int r;
 
 	bio_set_dev(bio, dd->dev->bdev);
-	bio->bi_iter.bi_sector = dd->start + dm_target_offset(ti, bio->bi_iter.bi_sector);
+	bio->bi_iter.bi_sector = dd->start + dust_sector;
 
 	if (bio_data_dir(bio) == READ)
-		r = dust_map_read(dd, bio->bi_iter.bi_sector, dd->fail_read_on_bb);
+		r = dust_map_read(dd, dust_sector, dd->fail_read_on_bb);
 	else
-		r = dust_map_write(dd, bio->bi_iter.bi_sector, dd->fail_read_on_bb);
+		r = dust_map_write(dd, dust_sector, dd->fail_read_on_bb);
 
 	return r;
 }
@@ -415,7 +415,7 @@ static int dust_message(struct dm_target *ti, unsigned int argc, char **argv,
 			char *result, unsigned int maxlen)
 {
 	struct dust_device *dd = ti->private;
-	sector_t size = bdev_nr_sectors(dd->dev->bdev);
+	sector_t size = dm_sector_div_up(ti->len, dd->sect_per_block);
 	bool invalid_msg = false;
 	int r = -EINVAL;
 	unsigned long long tmp, block;
@@ -462,8 +462,7 @@ static int dust_message(struct dm_target *ti, unsigned int argc, char **argv,
 			return r;
 
 		block = tmp;
-		sector_div(size, dd->sect_per_block);
-		if (block > size) {
+		if (block >= size) {
 			DMERR("selected block value out of range");
 			return r;
 		}
@@ -490,8 +489,7 @@ static int dust_message(struct dm_target *ti, unsigned int argc, char **argv,
 			return r;
 		}
 		wr_fail_cnt = tmp_ui;
-		sector_div(size, dd->sect_per_block);
-		if (block > size) {
+		if (block >= size) {
 			DMERR("selected block value out of range");
 			return r;
 		}

drivers/md/dm-era-target.c

@@ -1034,6 +1034,7 @@ static int metadata_checkpoint(struct era_metadata *md)
 static int metadata_take_snap(struct era_metadata *md)
 {
 	int r, inc;
+	dm_block_t location;
 	struct dm_block *clone;
 
 	if (md->metadata_snap != SUPERBLOCK_LOCATION) {
@@ -1071,7 +1072,9 @@ static int metadata_take_snap(struct era_metadata *md)
 	r = dm_sm_inc_block(md->sm, md->writeset_tree_root);
 	if (r) {
 		DMERR("%s: couldn't inc writeset tree root", __func__);
+		location = dm_block_location(clone);
 		dm_tm_unlock(md->tm, clone);
+		dm_sm_dec_block(md->sm, location);
 		return r;
 	}
 
@@ -1079,7 +1082,9 @@ static int metadata_take_snap(struct era_metadata *md)
 	if (r) {
 		DMERR("%s: couldn't inc era tree root", __func__);
 		dm_sm_dec_block(md->sm, md->writeset_tree_root);
+		location = dm_block_location(clone);
 		dm_tm_unlock(md->tm, clone);
+		dm_sm_dec_block(md->sm, location);
 		return r;
 	}
 

drivers/md/dm-inlinecrypt.c

@@ -347,7 +347,6 @@ static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	err = get_key_size(&argv[1]);
 	if (err < 0) {
 		ti->error = "Cannot parse key size";
-		err = -EINVAL;
 		goto bad;
 	}
 	ctx->key_size = err;

drivers/md/dm-integrity.c

@@ -66,6 +66,7 @@
 #define SB_VERSION_4			4
 #define SB_VERSION_5			5
 #define SB_VERSION_6			6
+#define SB_VERSION_7			7
 #define SB_SECTORS			8
 #define MAX_SECTORS_PER_BLOCK		8
 
@@ -91,6 +92,7 @@ struct superblock {
 #define SB_FLAG_FIXED_PADDING		0x8
 #define SB_FLAG_FIXED_HMAC		0x10
 #define SB_FLAG_INLINE			0x20
+#define SB_FLAG_DISCARD_KEYED		0x40
 
 #define	JOURNAL_ENTRY_ROUNDUP		8
 
@@ -277,6 +279,7 @@ struct dm_integrity_c {
 	bool recalculate_flag;
 	bool reset_recalculate_flag;
 	bool discard;
+	bool discard_keyed;
 	bool fix_padding;
 	bool fix_hmac;
 	bool legacy_recalculate;
@@ -483,7 +486,9 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr)
 
 static void sb_set_version(struct dm_integrity_c *ic)
 {
-	if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
+	if (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))
+		ic->sb->version = SB_VERSION_7;
+	else if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
 		ic->sb->version = SB_VERSION_6;
 	else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC))
 		ic->sb->version = SB_VERSION_5;
@@ -562,7 +567,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
 		}
 	}
 
-	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r))
 		return r;
 
@@ -1035,12 +1040,14 @@ static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned in
 		return crypt_journal(ic, encrypt, section, n_sections, comp);
 }
 
-static void complete_journal_io(unsigned long error, void *context)
+static void complete_journal_io(unsigned long error, unsigned long unsup, void *context)
 {
 	struct journal_completion *comp = context;
 
 	if (unlikely(error != 0))
 		dm_integrity_io_error(comp->ic, "writing journal", -EIO);
+	else if (unlikely(unsup != 0))
+		dm_integrity_io_error(comp->ic, "writing journal", -EOPNOTSUPP);
 	complete_journal_op(comp);
 }
 
@@ -1055,7 +1062,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
 
 	if (unlikely(dm_integrity_failed(ic))) {
 		if (comp)
-			complete_journal_io(-1UL, comp);
+			complete_journal_io(-1UL, -1UL, comp);
 		return;
 	}
 
@@ -1080,13 +1087,13 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
 	io_loc.sector = ic->start + SB_SECTORS + sector;
 	io_loc.count = n_sectors;
 
-	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r)) {
 		dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
 				      "reading journal" : "writing journal", r);
 		if (comp) {
 			WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
-			complete_journal_io(-1UL, comp);
+			complete_journal_io(-1UL, -1UL, comp);
 		}
 	}
 }
@@ -1177,7 +1184,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u
 	BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1));
 
 	if (unlikely(dm_integrity_failed(ic))) {
-		fn(-1UL, data);
+		fn(-1UL, -1UL, data);
 		return;
 	}
 
@@ -1197,10 +1204,10 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u
 	io_loc.sector = target;
 	io_loc.count = n_sectors;
 
-	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r)) {
 		WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
-		fn(-1UL, data);
+		fn(-1UL, -1UL, data);
 	}
 }
 
@@ -1414,7 +1421,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
 {
 	unsigned int hash_offset = 0;
 	unsigned char mismatch_hash = 0;
-	unsigned char mismatch_filler = !ic->discard;
+	unsigned char mismatch_filler = !ic->discard || ic->discard_keyed;
 
 	do {
 		unsigned char *data, *dp;
@@ -1466,7 +1473,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
 						}
 						hash_offset = 0;
 						mismatch_hash = 0;
-						mismatch_filler = !ic->discard;
+						mismatch_filler = !ic->discard || ic->discard_keyed;
 					}
 				}
 			}
@@ -1493,12 +1500,14 @@ struct flush_request {
 	struct completion comp;
 };
 
-static void flush_notify(unsigned long error, void *fr_)
+static void flush_notify(unsigned long error, unsigned long unsup, void *fr_)
 {
 	struct flush_request *fr = fr_;
 
 	if (unlikely(error != 0))
 		dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO);
+	else if (unlikely(unsup != 0))
+		dm_integrity_io_error(fr->ic, "flushing disk cache", -EOPNOTSUPP);
 	complete(&fr->comp);
 }
 
@@ -1521,7 +1530,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat
 		fr.io_reg.count = 0;
 		fr.ic = ic;
 		init_completion(&fr.comp);
-		r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
+		r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, NULL, IOPRIO_DEFAULT);
 		BUG_ON(r);
 	}
 
@@ -1642,7 +1651,8 @@ static void integrity_end_io(struct bio *bio)
 }
 
 static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector,
-					    const char *data, unsigned offset, char *result)
+					    const char *data, unsigned offset,
+					    unsigned int len, char *result)
 {
 	__le64 sector_le = cpu_to_le64(sector);
 	SHASH_DESC_ON_STACK(req, ic->internal_shash);
@@ -1671,10 +1681,12 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t
 		goto failed;
 	}
 
-	r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT);
-	if (unlikely(r < 0)) {
-		dm_integrity_io_error(ic, "crypto_shash_update", r);
-		goto failed;
+	if (likely(len)) {
+		r = crypto_shash_update(req, data + offset, len);
+		if (unlikely(r < 0)) {
+			dm_integrity_io_error(ic, "crypto_shash_update", r);
+			goto failed;
+		}
 	}
 
 	r = crypto_shash_final(req, result);
@@ -1695,7 +1707,8 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t
 }
 
 static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
-					    sector_t sector, struct page *page, unsigned offset, char *result)
+					    sector_t sector, struct page *page, unsigned offset,
+					    unsigned int len, char *result)
 {
 	__le64 sector_le = cpu_to_le64(sector);
 	struct ahash_request *req;
@@ -1704,6 +1717,7 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
 	int r;
 	unsigned int digest_size;
 	unsigned int nbytes = 0;
+	unsigned int nents = 1 + (len ? 1 : 0);
 
 	might_sleep();
 
@@ -1717,12 +1731,12 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait);
 
 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
-		sg_init_table(sg, 3);
+		sg_init_table(sg, nents + 1);
 		sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE);
 		nbytes += SALT_SIZE;
 		s++;
 	} else {
-		sg_init_table(sg, 2);
+		sg_init_table(sg, nents);
 	}
 
 	if (likely(!is_vmalloc_addr(&sector_le))) {
@@ -1735,8 +1749,10 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
 	nbytes += sizeof(sector_le);
 	s++;
 
-	sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset);
-	nbytes += ic->sectors_per_block << SECTOR_SHIFT;
+	if (likely(len)) {
+		sg_set_page(s, page, len, offset);
+		nbytes += len;
+	}
 
 	ahash_request_set_crypt(req, sg, result, nbytes);
 
@@ -1760,10 +1776,40 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
 static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
 				      sector_t sector, const char *data, unsigned offset, char *result)
 {
+	unsigned int len = ic->sectors_per_block << SECTOR_SHIFT;
+
 	if (likely(ic->internal_shash != NULL))
-		integrity_sector_checksum_shash(ic, sector, data, offset, result);
+		integrity_sector_checksum_shash(ic, sector, data, offset, len, result);
 	else
-		integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result);
+		integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data,
+						 offset, len, result);
+}
+
+/*
+ * Authenticated marker for a discarded block: HMAC_key(salt || sector), with
+ * no data payload. Because a real data tag's input always covers a full
+ * block, its length differs from this marker's, so the two can never
+ * collide structurally, regardless of block content.
+ */
+static void integrity_discard_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
+				       sector_t sector, char *result)
+{
+	if (likely(ic->internal_shash != NULL))
+		integrity_sector_checksum_shash(ic, sector, NULL, 0, 0, result);
+	else
+		integrity_sector_checksum_ahash(ic, ahash_req, sector, NULL, 0, 0, result);
+}
+
+static void integrity_discard_fill_tags(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
+					unsigned char *checksums, sector_t *sector,
+					unsigned int blocks)
+{
+	unsigned int i;
+
+	for (i = 0; i < blocks; i++) {
+		integrity_discard_checksum(ic, ahash_req, *sector, checksums + i * ic->tag_size);
+		*sector += ic->sectors_per_block;
+	}
 }
 
 static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p)
@@ -1792,6 +1838,29 @@ static void *integrity_identity(struct dm_integrity_c *ic, void *data)
 		return virt_to_page(data);
 }
 
+static int integrity_recheck_verify_tag(struct dm_integrity_io *dio, char *checksum,
+					char *on_disk_tag, sector_t logical_sector)
+{
+	struct dm_integrity_c *ic = dio->ic;
+	int r;
+
+	if (!ic->discard_keyed)
+		return dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
+					   &dio->metadata_offset, ic->tag_size, TAG_CMP);
+
+	r = dm_integrity_rw_tag(ic, on_disk_tag, &dio->metadata_block,
+				&dio->metadata_offset, ic->tag_size, TAG_READ);
+	if (unlikely(r))
+		return r;
+
+	r = crypto_memneq(on_disk_tag, checksum, ic->tag_size);
+	if (unlikely(r)) {
+		integrity_discard_checksum(ic, &dio->ahash_req, logical_sector, checksum);
+		r = crypto_memneq(on_disk_tag, checksum, ic->tag_size);
+	}
+	return r;
+}
+
 static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
 {
 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
@@ -1817,6 +1886,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
 			char *mem;
 			char *buffer = page_to_virt(page);
 			unsigned int buffer_offset;
+			char on_disk_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
 			int r;
 			struct dm_io_request io_req;
 			struct dm_io_region io_loc;
@@ -1837,15 +1907,15 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
 			buffer_offset = (sector - io_loc.sector) << SECTOR_SHIFT;
 			io_loc.count = round_up(io_loc.count, alignment);
 
-			r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+			r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 			if (unlikely(r)) {
 				dio->bi_status = errno_to_blk_status(r);
 				goto free_ret;
 			}
 
 			integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum);
-			r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
-						&dio->metadata_offset, ic->tag_size, TAG_CMP);
+			r = integrity_recheck_verify_tag(dio, checksum, on_disk_tag,
+							 logical_sector);
 			if (r) {
 				if (r > 0) {
 					DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
@@ -1911,13 +1981,18 @@ static void integrity_metadata(struct work_struct *w)
 			unsigned int bi_size = dio->bio_details.bi_iter.bi_size;
 			unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE;
 			unsigned int max_blocks = max_size / ic->tag_size;
+			sector_t sector = dio->range.logical_sector;
 
-			memset(checksums, DISCARD_FILLER, max_size);
+			if (!ic->discard_keyed)
+				memset(checksums, DISCARD_FILLER, max_size);
 
 			while (bi_size) {
 				unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
 
 				this_step_blocks = min(this_step_blocks, max_blocks);
+				if (ic->discard_keyed)
+					integrity_discard_fill_tags(ic, &dio->ahash_req, checksums,
+								    &sector, this_step_blocks);
 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
 							this_step_blocks * ic->tag_size, TAG_WRITE);
 				if (unlikely(r)) {
@@ -2782,7 +2857,8 @@ static void integrity_bio_wait(struct work_struct *w)
 		switch (r) {
 			case DM_MAPIO_KILL:
 				bio->bi_status = BLK_STS_IOERR;
-				fallthrough;
+				bio_endio(bio);
+				return;
 			case DM_MAPIO_REMAPPED:
 				submit_bio_noacct(bio);
 				fallthrough;
@@ -2890,7 +2966,7 @@ static void integrity_commit(struct work_struct *w)
 	}
 }
 
-static void complete_copy_from_journal(unsigned long error, void *context)
+static void complete_copy_from_journal(unsigned long error, unsigned long unsup, void *context)
 {
 	struct journal_io *io = context;
 	struct journal_completion *comp = io->comp;
@@ -2900,6 +2976,8 @@ static void complete_copy_from_journal(unsigned long error, void *context)
 	mempool_free(io, &ic->journal_io_mempool);
 	if (unlikely(error != 0))
 		dm_integrity_io_error(ic, "copying from journal", -EIO);
+	else if (unlikely(unsup != 0))
+		dm_integrity_io_error(ic, "copying from journal", -EOPNOTSUPP);
 	complete_journal_op(comp);
 }
 
@@ -3040,7 +3118,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start
 				r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
 							ic->tag_size, TAG_WRITE);
 				if (unlikely(r))
-					dm_integrity_io_error(ic, "reading tags", r);
+					dm_integrity_io_error(ic, "writing tags", r);
 			}
 
 			atomic_inc(&comp.in_flight);
@@ -3215,7 +3293,7 @@ static void integrity_recalc(struct work_struct *w)
 	io_loc.sector = get_data_sector(ic, area, offset);
 	io_loc.count = n_sectors;
 
-	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r)) {
 		dm_integrity_io_error(ic, "reading data", r);
 		goto err;
@@ -3792,6 +3870,8 @@ static void dm_integrity_resume(struct dm_target *ti)
 	ic->wrote_to_journal = false;
 
 	flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING);
+	if (ic->discard_keyed)
+		flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED);
 	r = sync_rw_sb(ic, REQ_OP_READ);
 	if (r)
 		dm_integrity_io_error(ic, "reading superblock", r);
@@ -3939,7 +4019,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 		arg_count += ic->sectors_per_block != 1;
 		arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
 		arg_count += ic->reset_recalculate_flag;
-		arg_count += ic->discard;
+		arg_count += ic->discard && !ic->discard_keyed;
+		arg_count += ic->discard_keyed;
 		arg_count += ic->mode != 'I'; /* interleave_sectors */
 		arg_count += ic->mode == 'J'; /* journal_sectors */
 		arg_count += ic->mode == 'J'; /* journal_watermark */
@@ -3962,8 +4043,10 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" recalculate");
 		if (ic->reset_recalculate_flag)
 			DMEMIT(" reset_recalculate");
-		if (ic->discard)
+		if (ic->discard && !ic->discard_keyed)
 			DMEMIT(" allow_discards");
+		if (ic->discard_keyed)
+			DMEMIT(" allow_discards_keyed");
 		if (ic->mode != 'I')
 			DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
 		DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
@@ -4013,6 +4096,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 		DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
 		       'y' : 'n');
 		DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
+		DMEMIT(",allow_discards_keyed=%c", ic->discard_keyed ? 'y' : 'n');
 		DMEMIT(",fix_padding=%c",
 		       ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
 		DMEMIT(",fix_hmac=%c",
@@ -4170,6 +4254,9 @@ static int initialize_superblock(struct dm_integrity_c *ic,
 		get_random_bytes(ic->sb->salt, SALT_SIZE);
 	}
 
+	if (ic->discard_keyed)
+		ic->sb->flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED);
+
 	if (!ic->meta_dev) {
 		if (ic->fix_padding)
 			ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING);
@@ -4634,6 +4721,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
 	if (!ic->journal_tree) {
 		*error = "Could not allocate memory for journal tree";
 		r = -ENOMEM;
+		goto bad;
 	}
 bad:
 	kfree(crypt_data);
@@ -4827,6 +4915,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
 			ic->reset_recalculate_flag = true;
 		} else if (!strcmp(opt_string, "allow_discards")) {
 			ic->discard = true;
+		} else if (!strcmp(opt_string, "allow_discards_keyed")) {
+			ic->discard = true;
+			ic->discard_keyed = true;
 		} else if (!strcmp(opt_string, "fix_padding")) {
 			ic->fix_padding = true;
 		} else if (!strcmp(opt_string, "fix_hmac")) {
@@ -4955,6 +5046,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
 		ti->error = "Discard can be only used with internal hash";
 		goto bad;
 	}
+	if (ic->discard_keyed && !ic->internal_hash_alg.key) {
+		r = -EINVAL;
+		ti->error = "Keyed discard can only be used with keyed internal hash";
+		goto bad;
+	}
 
 	ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
 	ic->autocommit_msec = sync_msec;
@@ -5073,7 +5169,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
 			should_write_sb = true;
 	}
 
-	if (!ic->sb->version || ic->sb->version > SB_VERSION_6) {
+	if (!ic->sb->version || ic->sb->version > SB_VERSION_7) {
 		r = -EINVAL;
 		ti->error = "Unknown version";
 		goto bad;
@@ -5121,6 +5217,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
 			goto bad;
 		}
 	}
+	if (!ic->discard_keyed && (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))) {
+		r = -EINVAL;
+		ti->error = "Keyed discard cannot be disabled once enabled";
+		goto bad;
+	}
 	if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
 		r = -EINVAL;
 		ti->error = "Journal mac mismatch";
@@ -5436,7 +5537,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
 
 static struct target_type integrity_target = {
 	.name			= "integrity",
-	.version		= {1, 14, 0},
+	.version		= {1, 15, 0},
 	.module			= THIS_MODULE,
 	.features		= DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
 	.ctr			= dm_integrity_ctr,

drivers/md/dm-io.c

@@ -33,6 +33,7 @@ struct dm_io_client {
  */
 struct io {
 	unsigned long error_bits;
+	unsigned long unsup_bits;
 	atomic_t count;
 	struct dm_io_client *client;
 	io_notify_fn callback;
@@ -119,6 +120,7 @@ static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,
 static void complete_io(struct io *io)
 {
 	unsigned long error_bits = io->error_bits;
+	unsigned long unsup_bits = io->unsup_bits;
 	io_notify_fn fn = io->callback;
 	void *context = io->context;
 
@@ -127,13 +129,17 @@ static void complete_io(struct io *io)
 					     io->vma_invalidate_size);
 
 	mempool_free(io, &io->client->pool);
-	fn(error_bits, context);
+	fn(error_bits, unsup_bits, context);
 }
 
 static void dec_count(struct io *io, unsigned int region, blk_status_t error)
 {
-	if (error)
-		set_bit(region, &io->error_bits);
+	if (unlikely(error)) {
+		if (error == BLK_STS_NOTSUPP || error == BLK_STS_INVAL)
+			set_bit(region, &io->unsup_bits);
+		else
+			set_bit(region, &io->error_bits);
+	}
 
 	if (atomic_dec_and_test(&io->count))
 		complete_io(io);
@@ -170,12 +176,11 @@ struct dpages {
 			 struct page **p, unsigned long *len, unsigned int *offset);
 	void (*next_page)(struct dpages *dp);
 
-	union {
-		unsigned int context_u;
-		struct bvec_iter context_bi;
-	};
+	unsigned int context_u;
 	void *context_ptr;
 
+	struct bio *orig_bio;
+
 	void *vma_invalidate_address;
 	unsigned long vma_invalidate_size;
 };
@@ -210,44 +215,6 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned int o
 	dp->context_ptr = pl;
 }
 
-/*
- * Functions for getting the pages from a bvec.
- */
-static void bio_get_page(struct dpages *dp, struct page **p,
-			 unsigned long *len, unsigned int *offset)
-{
-	struct bio_vec bvec = bvec_iter_bvec((struct bio_vec *)dp->context_ptr,
-					     dp->context_bi);
-
-	*p = bvec.bv_page;
-	*len = bvec.bv_len;
-	*offset = bvec.bv_offset;
-
-	/* avoid figuring it out again in bio_next_page() */
-	dp->context_bi.bi_sector = (sector_t)bvec.bv_len;
-}
-
-static void bio_next_page(struct dpages *dp)
-{
-	unsigned int len = (unsigned int)dp->context_bi.bi_sector;
-
-	bvec_iter_advance((struct bio_vec *)dp->context_ptr,
-			  &dp->context_bi, len);
-}
-
-static void bio_dp_init(struct dpages *dp, struct bio *bio)
-{
-	dp->get_page = bio_get_page;
-	dp->next_page = bio_next_page;
-
-	/*
-	 * We just use bvec iterator to retrieve pages, so it is ok to
-	 * access the bvec table directly here
-	 */
-	dp->context_ptr = bio->bi_io_vec;
-	dp->context_bi = bio->bi_iter;
-}
-
 /*
  * Functions for getting the pages from a VMA.
  */
@@ -332,6 +299,21 @@ static void do_region(const blk_opf_t opf, unsigned int region,
 		return;
 	}
 
+	if (dp->orig_bio) {
+		bio = bio_alloc_clone(where->bdev, dp->orig_bio, GFP_NOIO,
+				      &io->client->bios);
+		bio->bi_iter.bi_sector = where->sector;
+		bio->bi_iter.bi_size = where->count << SECTOR_SHIFT;
+		bio->bi_opf = opf;
+		bio->bi_end_io = endio;
+		bio->bi_ioprio = ioprio;
+		store_io_and_region_in_bio(bio, io, region);
+
+		atomic_inc(&io->count);
+		submit_bio(bio);
+		return;
+	}
+
 	/*
 	 * where->count may be zero if op holds a flush and we need to
 	 * send a zero-sized flush.
@@ -418,6 +400,7 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions,
 
 	io = mempool_alloc(&client->pool, GFP_NOIO);
 	io->error_bits = 0;
+	io->unsup_bits = 0;
 	atomic_set(&io->count, 1); /* see dispatch_io() */
 	io->client = client;
 	io->callback = fn;
@@ -431,20 +414,23 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions,
 
 struct sync_io {
 	unsigned long error_bits;
+	unsigned long unsup_bits;
 	struct completion wait;
 };
 
-static void sync_io_complete(unsigned long error, void *context)
+static void sync_io_complete(unsigned long error, unsigned long unsup, void *context)
 {
 	struct sync_io *sio = context;
 
 	sio->error_bits = error;
+	sio->unsup_bits = unsup;
 	complete(&sio->wait);
 }
 
 static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 		   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
-		   unsigned long *error_bits, unsigned short ioprio)
+		   unsigned long *error_bits, unsigned long *unsup_bits,
+		   unsigned short ioprio)
 {
 	struct sync_io sio;
 
@@ -457,8 +443,10 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 
 	if (error_bits)
 		*error_bits = sio.error_bits;
+	if (unsup_bits)
+		*unsup_bits = sio.unsup_bits;
 
-	return sio.error_bits ? -EIO : 0;
+	return sio.error_bits ? -EIO : sio.unsup_bits ? -EOPNOTSUPP : 0;
 }
 
 static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
@@ -468,6 +456,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 
 	dp->vma_invalidate_address = NULL;
 	dp->vma_invalidate_size = 0;
+	dp->orig_bio = NULL;
 
 	switch (io_req->mem.type) {
 	case DM_IO_PAGE_LIST:
@@ -475,7 +464,11 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 		break;
 
 	case DM_IO_BIO:
-		bio_dp_init(dp, io_req->mem.ptr.bio);
+		/*
+		 * The destination bios clone this bio's biovec directly, so
+		 * there are no per-page accessors to set up here.
+		 */
+		dp->orig_bio = io_req->mem.ptr.bio;
 		break;
 
 	case DM_IO_VMA:
@@ -500,7 +493,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 	  struct dm_io_region *where, unsigned long *sync_error_bits,
-	  unsigned short ioprio)
+	  unsigned long *sync_unsup_bits, unsigned short ioprio)
 {
 	int r;
 	struct dpages dp;
@@ -516,7 +509,8 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 
 	if (!io_req->notify.fn)
 		return sync_io(io_req->client, num_regions, where,
-			       io_req->bi_opf, &dp, sync_error_bits, ioprio);
+			       io_req->bi_opf, &dp, sync_error_bits,
+			       sync_unsup_bits, ioprio);
 
 	async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp,
 		 io_req->notify.fn, io_req->notify.context, ioprio);

drivers/md/dm-ioctl.c

@@ -54,10 +54,8 @@ struct hash_cell {
 };
 
 struct vers_iter {
-	size_t param_size;
 	struct dm_target_versions *vers, *old_vers;
 	char *end;
-	uint32_t flags;
 };
 
 
@@ -694,118 +692,108 @@ static int list_devices(struct file *filp, struct dm_ioctl *param, size_t param_
 {
 	struct rb_node *n;
 	struct hash_cell *hc;
-	size_t len, needed = 0;
-	struct gendisk *disk;
-	struct dm_name_list *orig_nl, *nl, *old_nl = NULL;
+	size_t len;
+	struct dm_name_list *nl, *old_nl = NULL;
+	void *result_start, *result_limit;
 	uint32_t *event_nr;
 
-	down_write(&_hash_lock);
-
-	/*
-	 * Loop through all the devices working out how much
-	 * space we need.
-	 */
-	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
-		hc = container_of(n, struct hash_cell, name_node);
-		if (!filter_device(hc, param->name, param->uuid))
-			continue;
-		needed += align_val(offsetof(struct dm_name_list, name) + strlen(hc->name) + 1);
-		needed += align_val(sizeof(uint32_t) * 2);
-		if (param->flags & DM_UUID_FLAG && hc->uuid)
-			needed += align_val(strlen(hc->uuid) + 1);
-	}
-
 	/*
 	 * Grab our output buffer.
 	 */
-	nl = orig_nl = get_result_buffer(param, param_size, &len);
-	if (len < needed || len < sizeof(nl->dev)) {
-		param->flags |= DM_BUFFER_FULL_FLAG;
-		goto out;
-	}
-	param->data_size = param->data_start + needed;
+	nl = result_start = get_result_buffer(param, param_size, &len);
+	result_limit = result_start + len;
 
-	nl->dev = 0;	/* Flags no data */
+	if (len >= sizeof(*nl))
+		nl->dev = 0;	/* Flags no data */
+
+	down_write(&_hash_lock);
 
 	/*
-	 * Now loop through filling out the names.
+	 * Loop through filling out the names.
 	 */
 	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
-		void *uuid_ptr;
+		void *next_nl;
 
 		hc = container_of(n, struct hash_cell, name_node);
 		if (!filter_device(hc, param->name, param->uuid))
 			continue;
-		if (old_nl)
-			old_nl->next = (uint32_t) ((void *) nl -
-						   (void *) old_nl);
-		disk = dm_disk(hc->md);
-		nl->dev = huge_encode_dev(disk_devt(disk));
-		nl->next = 0;
-		strcpy(nl->name, hc->name);
 
-		old_nl = nl;
-		event_nr = align_ptr(nl->name + strlen(hc->name) + 1);
+		len = strlen(hc->name);
+		event_nr = align_ptr(nl->name + len + 1);
+		next_nl = event_nr + 2;
+		if (next_nl > result_limit)
+			break;
+
+		((u64 *)event_nr)[-1] = 0;
+		memcpy(nl->name, hc->name, len);
+
+		nl->dev = huge_encode_dev(disk_devt(dm_disk(hc->md)));
+
 		event_nr[0] = dm_get_event_nr(hc->md);
 		event_nr[1] = 0;
-		uuid_ptr = align_ptr(event_nr + 2);
+
 		if (param->flags & DM_UUID_FLAG) {
 			if (hc->uuid) {
+				len = strlen(hc->uuid);
+				next_nl = align_ptr(next_nl + len + 1);
+				if (next_nl > result_limit)
+					break;
 				event_nr[1] |= DM_NAME_LIST_FLAG_HAS_UUID;
-				strcpy(uuid_ptr, hc->uuid);
-				uuid_ptr = align_ptr(uuid_ptr + strlen(hc->uuid) + 1);
+				((u64 *)next_nl)[-1] = 0;
+				memcpy(event_nr + 2, hc->uuid, len);
 			} else {
 				event_nr[1] |= DM_NAME_LIST_FLAG_DOESNT_HAVE_UUID;
 			}
 		}
-		nl = uuid_ptr;
+		nl->next = next_nl - (void *)nl;
+		old_nl = nl;
+		nl = next_nl;
 	}
-	/*
-	 * If mismatch happens, security may be compromised due to buffer
-	 * overflow, so it's better to crash.
-	 */
-	BUG_ON((char *)nl - (char *)orig_nl != needed);
 
- out:
+	if (old_nl)
+		old_nl->next = 0;
+
+	if (n)
+		param->flags |= DM_BUFFER_FULL_FLAG;
+	else
+		param->data_size = param->data_start + ((void *)nl - result_start);
+
 	up_write(&_hash_lock);
 	return 0;
 }
 
-static void list_version_get_needed(struct target_type *tt, void *needed_param)
-{
-	size_t *needed = needed_param;
-
-	*needed += sizeof(struct dm_target_versions);
-	*needed += strlen(tt->name) + 1;
-	*needed += ALIGN_MASK;
-}
-
 static void list_version_get_info(struct target_type *tt, void *param)
 {
 	struct vers_iter *info = param;
+	struct dm_target_versions *vers = info->vers;
+	size_t name_len = strlen(tt->name);
 
-	/* Check space - it might have changed since the first iteration */
-	if ((char *)info->vers + sizeof(struct dm_target_versions) + strlen(tt->name) + 1 > info->end) {
-		info->flags = DM_BUFFER_FULL_FLAG;
+	if (!vers)
+		return;
+
+	info->old_vers = vers;
+	info->vers = align_ptr((void *)(info->vers + 1) + name_len + 1);
+
+	/* Check space */
+	if ((char *)info->vers > info->end) {
+		info->vers = NULL;
 		return;
 	}
 
-	if (info->old_vers)
-		info->old_vers->next = (uint32_t) ((void *)info->vers - (void *)info->old_vers);
+	/* Zero padding and terminate vers->name[] */
+	((u64 *)info->vers)[-1] = 0;
 
-	info->vers->version[0] = tt->version[0];
-	info->vers->version[1] = tt->version[1];
-	info->vers->version[2] = tt->version[2];
-	info->vers->next = 0;
-	strcpy(info->vers->name, tt->name);
+	vers->next = (char *)info->vers - (char *)vers;
 
-	info->old_vers = info->vers;
-	info->vers = align_ptr((void *)(info->vers + 1) + strlen(tt->name) + 1);
+	vers->version[0] = tt->version[0];
+	vers->version[1] = tt->version[1];
+	vers->version[2] = tt->version[2];
+	memcpy(vers->name, tt->name, name_len);
 }
 
 static int __list_versions(struct dm_ioctl *param, size_t param_size, const char *name)
 {
-	size_t len, needed = 0;
+	size_t len;
 	struct dm_target_versions *vers;
 	struct vers_iter iter_info;
 	struct target_type *tt = NULL;
@@ -816,41 +804,31 @@ static int __list_versions(struct dm_ioctl *param, size_t param_size, const char
 			return -EINVAL;
 	}
 
-	/*
-	 * Loop through all the devices working out how much
-	 * space we need.
-	 */
-	if (!tt)
-		dm_target_iterate(list_version_get_needed, &needed);
-	else
-		list_version_get_needed(tt, &needed);
-
 	/*
 	 * Grab our output buffer.
 	 */
 	vers = get_result_buffer(param, param_size, &len);
-	if (len < needed) {
-		param->flags |= DM_BUFFER_FULL_FLAG;
-		goto out;
-	}
-	param->data_size = param->data_start + needed;
 
-	iter_info.param_size = param_size;
 	iter_info.old_vers = NULL;
 	iter_info.vers = vers;
-	iter_info.flags = 0;
-	iter_info.end = (char *)vers + needed;
+	iter_info.end = (char *)vers + len;
 
 	/*
-	 * Now loop through filling out the names & versions.
+	 * Loop through filling out the names & versions.
 	 */
 	if (!tt)
 		dm_target_iterate(list_version_get_info, &iter_info);
 	else
 		list_version_get_info(tt, &iter_info);
-	param->flags |= iter_info.flags;
 
- out:
+	if (iter_info.vers) {
+		if (iter_info.old_vers)
+			iter_info.old_vers->next = 0;
+		param->data_size = param->data_start + ((char *)iter_info.vers - (char *)vers);
+	} else {
+		param->flags |= DM_BUFFER_FULL_FLAG;
+	}
+
 	if (tt)
 		dm_put_target_type(tt);
 	return 0;
@@ -1444,10 +1422,6 @@ static void retrieve_status(struct dm_table *table,
 
 		outptr += sizeof(struct dm_target_spec);
 		remaining = len - (outptr - outbuf);
-		if (remaining <= 0) {
-			param->flags |= DM_BUFFER_FULL_FLAG;
-			break;
-		}
 
 		/* Get the status/table string from the target driver */
 		if (ti->type->status) {
@@ -1998,33 +1972,32 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para
 static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
 {
 	static const struct {
-		int cmd;
 		int flags;
 		ioctl_fn fn;
 	} _ioctls[] = {
-		{DM_VERSION_CMD, 0, NULL}, /* version is dealt with elsewhere */
-		{DM_REMOVE_ALL_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, remove_all},
-		{DM_LIST_DEVICES_CMD, 0, list_devices},
+		[DM_VERSION_CMD] = {0, NULL}, /* version is dealt with elsewhere */
+		[DM_REMOVE_ALL_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, remove_all},
+		[DM_LIST_DEVICES_CMD] = {0, list_devices},
 
-		{DM_DEV_CREATE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_create},
-		{DM_DEV_REMOVE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_remove},
-		{DM_DEV_RENAME_CMD, IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_rename},
-		{DM_DEV_SUSPEND_CMD, IOCTL_FLAGS_NO_PARAMS, dev_suspend},
-		{DM_DEV_STATUS_CMD, IOCTL_FLAGS_NO_PARAMS, dev_status},
-		{DM_DEV_WAIT_CMD, 0, dev_wait},
+		[DM_DEV_CREATE_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_create},
+		[DM_DEV_REMOVE_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_remove},
+		[DM_DEV_RENAME_CMD] = {IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_rename},
+		[DM_DEV_SUSPEND_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_suspend},
+		[DM_DEV_STATUS_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_status},
+		[DM_DEV_WAIT_CMD] = {0, dev_wait},
 
-		{DM_TABLE_LOAD_CMD, 0, table_load},
-		{DM_TABLE_CLEAR_CMD, IOCTL_FLAGS_NO_PARAMS, table_clear},
-		{DM_TABLE_DEPS_CMD, 0, table_deps},
-		{DM_TABLE_STATUS_CMD, 0, table_status},
+		[DM_TABLE_LOAD_CMD] = {0, table_load},
+		[DM_TABLE_CLEAR_CMD] = {IOCTL_FLAGS_NO_PARAMS, table_clear},
+		[DM_TABLE_DEPS_CMD] = {0, table_deps},
+		[DM_TABLE_STATUS_CMD] = {0, table_status},
 
-		{DM_LIST_VERSIONS_CMD, 0, list_versions},
+		[DM_LIST_VERSIONS_CMD] = {0, list_versions},
 
-		{DM_TARGET_MSG_CMD, 0, target_message},
-		{DM_DEV_SET_GEOMETRY_CMD, 0, dev_set_geometry},
-		{DM_DEV_ARM_POLL_CMD, IOCTL_FLAGS_NO_PARAMS, dev_arm_poll},
-		{DM_GET_TARGET_VERSION_CMD, 0, get_target_version},
-		{DM_MPATH_PROBE_PATHS_CMD, 0, NULL}, /* block device ioctl */
+		[DM_TARGET_MSG_CMD] = {0, target_message},
+		[DM_DEV_SET_GEOMETRY_CMD] = {0, dev_set_geometry},
+		[DM_DEV_ARM_POLL_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_arm_poll},
+		[DM_GET_TARGET_VERSION_CMD] = {0, get_target_version},
+		[DM_MPATH_PROBE_PATHS_CMD] = {0, NULL}, /* block device ioctl */
 	};
 
 	if (unlikely(cmd >= ARRAY_SIZE(_ioctls)))
@@ -2273,12 +2246,9 @@ static long dm_compat_ctl_ioctl(struct file *file, uint command, ulong u)
 
 static int dm_open(struct inode *inode, struct file *filp)
 {
-	int r;
 	struct dm_file *priv;
 
-	r = nonseekable_open(inode, filp);
-	if (unlikely(r))
-		return r;
+	nonseekable_open(inode, filp);
 
 	priv = filp->private_data = kmalloc_obj(struct dm_file);
 	if (!priv)

drivers/md/dm-kcopyd.c

@@ -517,16 +517,16 @@ static int run_complete_job(struct kcopyd_job *job)
 	return 0;
 }
 
-static void complete_io(unsigned long error, void *context)
+static void complete_io(unsigned long error, unsigned long unsup, void *context)
 {
 	struct kcopyd_job *job = context;
 	struct dm_kcopyd_client *kc = job->kc;
 
 	io_job_finish(kc->throttle);
 
-	if (error) {
+	if (unlikely((error | unsup) != 0)) {
 		if (op_is_write(job->op))
-			job->write_err |= error;
+			job->write_err |= error | unsup;
 		else
 			job->read_err = 1;
 
@@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job)
 	io_job_start(job->kc->throttle);
 
 	if (job->op == REQ_OP_READ)
-		r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT);
+		r = dm_io(&io_req, 1, &job->source, NULL, NULL, IOPRIO_DEFAULT);
 	else
-		r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT);
+		r = dm_io(&io_req, job->num_dests, job->dests, NULL, NULL, IOPRIO_DEFAULT);
 
 	return r;
 }

drivers/md/dm-log-userspace-base.c

@@ -139,6 +139,7 @@ static int build_constructor_string(struct dm_target *ti,
 		str_size += strlen(argv[i]) + 1; /* +1 for space between args */
 
 	str_size += 20; /* Max number of chars in a printed u64 number */
+	str_size++; /* For NUL-terminator */
 
 	str = kzalloc(str_size, GFP_KERNEL);
 	if (!str) {

drivers/md/dm-log.c

@@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op)
 {
 	lc->io_req.bi_opf = op;
 
-	return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT);
+	return dm_io(&lc->io_req, 1, &lc->header_location, NULL, NULL, IOPRIO_DEFAULT);
 }
 
 static int flush_header(struct log_c *lc)
@@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc)
 
 	lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
-	return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT);
+	return dm_io(&lc->io_req, 1, &null_location, NULL, NULL, IOPRIO_DEFAULT);
 }
 
 static int read_header(struct log_c *log)

drivers/md/dm-pcache/cache.c

@@ -118,7 +118,14 @@ int cache_pos_decode(struct pcache_cache *cache,
 	if (!latest_addr)
 		return -EIO;
 
+	if (!cache_seg_id_valid(cache, latest.cache_seg_id))
+		return -EIO;
+
 	pos->cache_seg = &cache->segments[latest.cache_seg_id];
+
+	if (latest.seg_off >= pos->cache_seg->segment.data_size)
+		return -EIO;
+
 	pos->seg_off = latest.seg_off;
 	*seq = latest.header.seq;
 	*index = (latest_addr - pos_onmedia);
@@ -155,6 +162,7 @@ static int cache_init(struct dm_pcache *pcache)
 	cache->cache_dev = &pcache->cache_dev;
 	cache->n_segs = cache_dev->seg_num;
 	atomic_set(&cache->gc_errors, 0);
+	atomic_set(&cache->writeback_errors, 0);
 	spin_lock_init(&cache->seg_map_lock);
 	spin_lock_init(&cache->key_head_lock);
 
@@ -194,6 +202,7 @@ static int cache_tail_init(struct pcache_cache *cache)
 {
 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
+	int ret;
 
 	if (new_cache) {
 		__set_bit(0, cache->seg_map);
@@ -210,6 +219,12 @@ static int cache_tail_init(struct pcache_cache *cache)
 			pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n");
 			return -EIO;
 		}
+
+		ret = cache_verify_dirty_tail(cache);
+		if (ret) {
+			pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n");
+			return ret;
+		}
 	}
 
 	return 0;
@@ -247,6 +262,13 @@ static int get_seg_id(struct pcache_cache *cache,
 		} else {
 			*seg_id = cache->cache_info.seg_id;
 		}
+
+		if (*seg_id >= cache_dev->seg_num) {
+			pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n",
+				       *seg_id, cache_dev->seg_num);
+			ret = -EIO;
+			goto err;
+		}
 	}
 	return 0;
 err:
@@ -262,6 +284,13 @@ static int cache_segs_init(struct pcache_cache *cache)
 	int ret;
 	u32 i;
 
+	if (cache_info->n_segs > cache->cache_dev->seg_num) {
+		pcache_dev_err(CACHE_TO_PCACHE(cache),
+			       "cache_info n_segs %u exceeds cache device segments %u\n",
+			       cache_info->n_segs, cache->cache_dev->seg_num);
+		return -EIO;
+	}
+
 	for (i = 0; i < cache_info->n_segs; i++) {
 		ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);
 		if (ret)

drivers/md/dm-pcache/cache.h

@@ -180,6 +180,7 @@ struct pcache_cache {
 		u32 advance;
 		int ret;
 	} writeback_ctx;
+	atomic_t		writeback_errors;
 
 	char gc_kset_onmedia_buf[PCACHE_KSET_ONMEDIA_SIZE_MAX];
 	struct delayed_work	gc_work;
@@ -273,7 +274,7 @@ struct pcache_cache_subtree_walk_ctx {
 	struct list_head *submit_req_list;
 
 	/*
-	 *	  |--------|		key_tmp
+	 *        |--------|		key_tmp
 	 * |====|			key
 	 */
 	int (*before)(struct pcache_cache_key *key, struct pcache_cache_key *key_tmp,
@@ -281,7 +282,7 @@ struct pcache_cache_subtree_walk_ctx {
 
 	/*
 	 * |----------|			key_tmp
-	 *		|=====|		key
+	 *              |=====|		key
 	 */
 	int (*after)(struct pcache_cache_key *key, struct pcache_cache_key *key_tmp,
 			struct pcache_cache_subtree_walk_ctx *ctx);
@@ -340,7 +341,6 @@ void cache_seg_set_next_seg(struct pcache_cache_segment *cache_seg, u32 seg_id);
 
 /* cache request*/
 int pcache_cache_flush(struct pcache_cache *cache);
-void miss_read_end_work_fn(struct work_struct *work);
 int pcache_cache_handle_req(struct pcache_cache *cache, struct pcache_request *pcache_req);
 
 /* gc */
@@ -420,6 +420,20 @@ static inline bool cache_seg_is_ctrl_seg(u32 cache_seg_id)
 	return (cache_seg_id == 0);
 }
 
+/**
+ * cache_seg_id_valid - Validate a cache segment id read from the cache device.
+ * @cache: Pointer to the pcache_cache structure.
+ * @cache_seg_id: Segment id decoded from on-media metadata.
+ *
+ * On-media segment ids are only protected by a CRC, which an attacker who can
+ * format the cache device computes over their chosen value. Reject any id that
+ * would index cache->segments[] out of bounds before it is dereferenced.
+ */
+static inline bool cache_seg_id_valid(struct pcache_cache *cache, u32 cache_seg_id)
+{
+	return cache_seg_id < cache->cache_info.n_segs;
+}
+
 /**
  * cache_key_cutfront - Cuts a specified length from the front of a cache key.
  * @key: Pointer to pcache_cache_key structure.
@@ -491,6 +505,27 @@ static inline u32 cache_key_data_crc(struct pcache_cache_key *key)
 	return crc32c(PCACHE_CRC_SEED, data, key->len);
 }
 
+/**
+ * kset_onmedia_valid - Validate a kset header read from the cache device.
+ * @kset_onmedia: Pointer to the kset copied from on-media metadata.
+ *
+ * The magic and CRC are attacker-computable (fixed public seed). A non-last
+ * kset stores key_num keys inline, and cache_kset_crc() and the replay loop
+ * read struct_size(.., data, key_num) bytes from a buffer sized for
+ * PCACHE_KSET_KEYS_MAX keys, so key_num must be bounded before any such use.
+ */
+static inline bool kset_onmedia_valid(struct pcache_cache_kset_onmedia *kset_onmedia)
+{
+	if (kset_onmedia->magic != PCACHE_KSET_MAGIC)
+		return false;
+
+	if (!(kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) &&
+	    kset_onmedia->key_num > PCACHE_KSET_KEYS_MAX)
+		return false;
+
+	return true;
+}
+
 static inline u32 cache_kset_crc(struct pcache_cache_kset_onmedia *kset_onmedia)
 {
 	u32 crc_size;
@@ -630,6 +665,8 @@ static inline int cache_decode_dirty_tail(struct pcache_cache *cache)
 				&cache->dirty_tail_index);
 }
 
+int cache_verify_dirty_tail(struct pcache_cache *cache);
+
 int pcache_cache_init(void);
 void pcache_cache_exit(void);
 #endif /* _PCACHE_CACHE_H */

drivers/md/dm-pcache/cache_dev.c

@@ -242,6 +242,8 @@ int cache_dev_start(struct dm_pcache *pcache)
 	struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
 	struct pcache_sb sb;
 	bool format = false;
+	u32 seg_num;
+	u64 max_segs;
 	int ret;
 
 	mutex_init(&cache_dev->seg_lock);
@@ -269,7 +271,25 @@ int cache_dev_start(struct dm_pcache *pcache)
 		goto dax_release;
 
 	cache_dev->sb_flags = le32_to_cpu(sb.flags);
-	ret = cache_dev_init(cache_dev, le32_to_cpu(sb.seg_num));
+
+	/*
+	 * seg_num is read from the crc32c-only superblock, so whoever supplies
+	 * the cache device controls it. It is the ceiling every later on-media
+	 * segment id is validated against, so bound it against what the device
+	 * physically holds before it is trusted, or a forged seg_num lets a
+	 * segment id address past the DAX mapping.
+	 */
+	seg_num = le32_to_cpu(sb.seg_num);
+	max_segs = (bdev_nr_bytes(cache_dev->dm_dev->bdev) - PCACHE_SEGMENTS_OFF) /
+		   PCACHE_SEG_SIZE;
+	if (seg_num == 0 || seg_num > max_segs || seg_num > PCACHE_CACHE_SEGS_MAX) {
+		pcache_dev_err(pcache, "invalid seg_num %u from cache device (device holds %llu, max %u)\n",
+			       seg_num, max_segs, (u32)PCACHE_CACHE_SEGS_MAX);
+		ret = -EIO;
+		goto dax_release;
+	}
+
+	ret = cache_dev_init(cache_dev, seg_num);
 	if (ret)
 		goto dax_release;
 

drivers/md/dm-pcache/cache_gc.c

@@ -6,14 +6,13 @@
 
 /**
  * cache_key_gc - Releases the reference of a cache key segment.
- * @cache: Pointer to the pcache_cache structure.
  * @key: Pointer to the cache key to be garbage collected.
  *
  * This function decrements the reference count of the cache segment
  * associated with the given key. If the reference count drops to zero,
  * the segment may be invalidated and reused.
  */
-static void cache_key_gc(struct pcache_cache *cache, struct pcache_cache_key *key)
+static void cache_key_gc(struct pcache_cache_key *key)
 {
 	cache_seg_put(key->cache_pos.cache_seg);
 }
@@ -37,18 +36,18 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t
 
 	kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->gc_kset_onmedia_buf;
 
-	to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - key_tail->seg_off);
+	to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(key_tail));
 	ret = copy_mc_to_kernel(kset_onmedia, key_addr, to_copy);
 	if (ret) {
 		pcache_dev_err(pcache, "error to read kset: %d", ret);
 		return false;
 	}
 
-	/* Check if kset_onmedia is corrupted */
-	if (kset_onmedia->magic != PCACHE_KSET_MAGIC) {
-		pcache_dev_debug(pcache, "gc error: magic is not as expected. key_tail: %u:%u magic: %llx, expected: %llx\n",
+	/* Reject a corrupted or out-of-bounds kset before reading its keys */
+	if (!kset_onmedia_valid(kset_onmedia)) {
+		pcache_dev_debug(pcache, "gc error: invalid kset. key_tail: %u:%u magic: %llx, key_num: %u\n",
 					key_tail->cache_seg->cache_seg_id, key_tail->seg_off,
-					kset_onmedia->magic, PCACHE_KSET_MAGIC);
+					kset_onmedia->magic, kset_onmedia->key_num);
 		return false;
 	}
 
@@ -74,11 +73,17 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t
  * @cache: Pointer to the pcache_cache structure.
  * @kset_onmedia: Pointer to the kset_onmedia structure for the last kset.
  */
-static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia)
+static int last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia)
 {
 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
 	struct pcache_cache_segment *cur_seg, *next_seg;
 
+	if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) {
+		pcache_dev_err(pcache, "invalid next_cache_seg_id %u in gc (n_segs %u)\n",
+				kset_onmedia->next_cache_seg_id, cache->n_segs);
+		return -EIO;
+	}
+
 	cur_seg = cache->key_tail.cache_seg;
 
 	next_seg = &cache->segments[kset_onmedia->next_cache_seg_id];
@@ -94,6 +99,8 @@ static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_on
 	spin_lock(&cache->seg_map_lock);
 	__clear_bit(cur_seg->cache_seg_id, cache->seg_map);
 	spin_unlock(&cache->seg_map_lock);
+
+	return 0;
 }
 
 void pcache_cache_gc_fn(struct work_struct *work)
@@ -130,10 +137,19 @@ void pcache_cache_gc_fn(struct work_struct *work)
 			if (dirty_tail.cache_seg == key_tail.cache_seg)
 				break;
 
-			last_kset_gc(cache, kset_onmedia);
+			ret = last_kset_gc(cache, kset_onmedia);
+			if (ret) {
+				atomic_inc(&cache->gc_errors);
+				return;
+			}
 			continue;
 		}
 
+		if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&key_tail)) {
+			atomic_inc(&cache->gc_errors);
+			return;
+		}
+
 		for (i = 0; i < kset_onmedia->key_num; i++) {
 			struct pcache_cache_key key_tmp = { 0 };
 
@@ -152,7 +168,7 @@ void pcache_cache_gc_fn(struct work_struct *work)
 				return;
 			}
 
-			cache_key_gc(cache, key);
+			cache_key_gc(key);
 		}
 
 		pcache_dev_debug(pcache, "gc advance: %u:%u %u\n",

drivers/md/dm-pcache/cache_key.c

@@ -90,13 +90,36 @@ int cache_key_decode(struct pcache_cache *cache,
 			struct pcache_cache_key *key)
 {
 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
+	u64 dev_bytes = (u64)cache->dev_size << SECTOR_SHIFT;
 
 	key->off = key_onmedia->off;
 	key->len = key_onmedia->len;
 
+	if (key_onmedia->len == 0 ||
+	    key_onmedia->len > dev_bytes ||
+	    key_onmedia->off > dev_bytes - key_onmedia->len) {
+		pcache_dev_err(pcache, "key off %llu + len %u exceeds device size\n",
+				key_onmedia->off, key_onmedia->len);
+		return -EIO;
+	}
+
+	if (!cache_seg_id_valid(cache, key_onmedia->cache_seg_id)) {
+		pcache_dev_err(pcache, "invalid cache_seg_id %u in cache key (n_segs %u)\n",
+				key_onmedia->cache_seg_id, cache->n_segs);
+		return -EIO;
+	}
+
 	key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id];
 	key->cache_pos.seg_off = key_onmedia->cache_seg_off;
 
+	if ((u64)key->cache_pos.seg_off + key->len >
+			key->cache_pos.cache_seg->segment.data_size) {
+		pcache_dev_err(pcache, "key seg_off %u + len %u exceeds segment data size %u\n",
+				key->cache_pos.seg_off, key->len,
+				key->cache_pos.cache_seg->segment.data_size);
+		return -EIO;
+	}
+
 	key->seg_gen = key_onmedia->seg_gen;
 	key->flags = key_onmedia->flags;
 
@@ -265,7 +288,7 @@ int cache_subtree_walk(struct pcache_cache_subtree_walk_ctx *ctx)
 
 		/*
 		 * If key_tmp starts after the end of key, stop traversing.
-		 *	  |--------|
+		 *        |--------|
 		 * |====|
 		 */
 		if (cache_key_lstart(key_tmp) >= cache_key_lend(key)) {
@@ -728,18 +751,17 @@ static int kset_replay(struct pcache_cache *cache, struct pcache_cache_kset_onme
 			goto err;
 		}
 
-		__set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map);
-
 		/* Check if the segment generation is valid for insertion. */
 		if (key->seg_gen < key->cache_pos.cache_seg->gen) {
 			cache_key_put(key);
-		} else {
-			cache_subtree = get_subtree(&cache->req_key_tree, key->off);
-			spin_lock(&cache_subtree->tree_lock);
-			cache_key_insert(&cache->req_key_tree, key, true);
-			spin_unlock(&cache_subtree->tree_lock);
+			continue;
 		}
 
+		__set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map);
+		cache_subtree = get_subtree(&cache->req_key_tree, key->off);
+		spin_lock(&cache_subtree->tree_lock);
+		cache_key_insert(&cache->req_key_tree, key, true);
+		spin_unlock(&cache_subtree->tree_lock);
 		cache_seg_get(key->cache_pos.cache_seg);
 	}
 
@@ -754,7 +776,7 @@ int cache_replay(struct pcache_cache *cache)
 	struct pcache_cache_pos pos_tail;
 	struct pcache_cache_pos *pos;
 	struct pcache_cache_kset_onmedia *kset_onmedia;
-	u32 to_copy, count = 0;
+	u32 to_copy, count = 0, last_hops = 0;
 	int ret = 0;
 
 	kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL);
@@ -771,14 +793,14 @@ int cache_replay(struct pcache_cache *cache)
 	__set_bit(pos->cache_seg->cache_seg_id, cache->seg_map);
 
 	while (true) {
-		to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - pos->seg_off);
+		to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(pos));
 		ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(pos), to_copy);
 		if (ret) {
 			ret = -EIO;
 			goto out;
 		}
 
-		if (kset_onmedia->magic != PCACHE_KSET_MAGIC ||
+		if (!kset_onmedia_valid(kset_onmedia) ||
 				kset_onmedia->crc != cache_kset_crc(kset_onmedia)) {
 			break;
 		}
@@ -789,6 +811,16 @@ int cache_replay(struct pcache_cache *cache)
 
 			pcache_dev_debug(pcache, "last kset replay, next: %u\n", kset_onmedia->next_cache_seg_id);
 
+			if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) {
+				ret = -EIO;
+				goto out;
+			}
+
+			if (++last_hops > cache->n_segs) {
+				ret = -EIO;
+				goto out;
+			}
+
 			next_seg = &cache->segments[kset_onmedia->next_cache_seg_id];
 
 			pos->cache_seg = next_seg;
@@ -799,6 +831,11 @@ int cache_replay(struct pcache_cache *cache)
 		}
 
 		/* Replay the kset and check for errors. */
+		if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(pos)) {
+			ret = -EIO;
+			goto out;
+		}
+
 		ret = kset_replay(cache, kset_onmedia);
 		if (ret)
 			goto out;
@@ -820,6 +857,75 @@ int cache_replay(struct pcache_cache *cache)
 	return ret;
 }
 
+/*
+ * cache_verify_dirty_tail - reject a persisted dirty_tail whose last-kset
+ * chain does not terminate.
+ *
+ * dirty_tail is decoded independently of the key_tail chain cache_replay()
+ * walks, so replay's hop cap does not cover it. A crafted chain that loops
+ * back on itself makes the writeback worker re-arm forever; walk it once here
+ * with the same cap and fail the load if it does not end within n_segs hops.
+ */
+int cache_verify_dirty_tail(struct pcache_cache *cache)
+{
+	struct pcache_cache_pos pos;
+	struct pcache_cache_kset_onmedia *kset_onmedia;
+	u32 to_copy, last_hops = 0, count = 0;
+	int ret = 0;
+
+	kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL);
+	if (!kset_onmedia)
+		return -ENOMEM;
+
+	cache_pos_copy(&pos, &cache->dirty_tail);
+
+	while (true) {
+		to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(&pos));
+		ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(&pos), to_copy);
+		if (ret) {
+			ret = -EIO;
+			goto out;
+		}
+
+		/* A missing, short or corrupt kset is the normal end of the chain. */
+		if (!kset_onmedia_valid(kset_onmedia) ||
+		    kset_onmedia->crc != cache_kset_crc(kset_onmedia)) {
+			ret = 0;
+			goto out;
+		}
+
+		if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) {
+			if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) {
+				ret = -EIO;
+				goto out;
+			}
+
+			if (++last_hops > cache->n_segs) {
+				ret = -EIO;
+				goto out;
+			}
+
+			pos.cache_seg = &cache->segments[kset_onmedia->next_cache_seg_id];
+			pos.seg_off = 0;
+			continue;
+		}
+
+		if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&pos)) {
+			ret = -EIO;
+			goto out;
+		}
+
+		cache_pos_advance(&pos, get_kset_onmedia_size(kset_onmedia));
+		if (++count > 512) {
+			cond_resched();
+			count = 0;
+		}
+	}
+out:
+	kfree(kset_onmedia);
+	return ret;
+}
+
 int cache_tree_init(struct pcache_cache *cache, struct pcache_cache_tree *cache_tree, u32 n_subtrees)
 {
 	int ret;

drivers/md/dm-pcache/cache_req.c

@@ -39,13 +39,11 @@ static int cache_data_alloc(struct pcache_cache *cache, struct pcache_cache_key
 	struct pcache_cache_pos *head_pos;
 	struct pcache_cache_segment *cache_seg;
 	u32 seg_remain;
-	u32 allocated = 0, to_alloc;
 	int ret = 0;
 
 	preempt_disable();
 	data_head = get_data_head(cache);
 again:
-	to_alloc = key->len - allocated;
 	if (!data_head->head_pos.cache_seg) {
 		seg_remain = 0;
 	} else {
@@ -57,10 +55,9 @@ static int cache_data_alloc(struct pcache_cache *cache, struct pcache_cache_key
 		seg_remain = cache_seg_remain(head_pos);
 	}
 
-	if (seg_remain > to_alloc) {
+	if (seg_remain > key->len) {
 		/* If remaining space in segment is sufficient for the cache key, allocate it. */
-		cache_pos_advance(head_pos, to_alloc);
-		allocated += to_alloc;
+		cache_pos_advance(head_pos, key->len);
 		cache_seg_get(cache_seg);
 	} else if (seg_remain) {
 		/* If remaining space is not enough, allocate the remaining space and adjust the cache key length. */
@@ -317,7 +314,7 @@ static struct pcache_backing_dev_req *get_pre_alloc_req(struct pcache_cache_subt
  *
  * The scenario handled here:
  *
- *	  |--------|			  key_tmp (existing cached range)
+ *        |--------|			  key_tmp (existing cached range)
  * |====|					   key (requested range, preceding key_tmp)
  *
  * Since `key` is before `key_tmp`, it signifies that the requested data
@@ -352,7 +349,7 @@ static int read_before(struct pcache_cache_key *key, struct pcache_cache_key *ke
  * During cache_subtree_walk, this function manages a scenario where part of the
  * requested data range overlaps with an existing cache node (`key_tmp`).
  *
- *	 |----------------|  key_tmp (existing cached range)
+ *     |----------------|  key_tmp (existing cached range)
  * |===========|		   key (requested range, overlapping the tail of key_tmp)
  */
 static int read_overlap_tail(struct pcache_cache_key *key, struct pcache_cache_key *key_tmp,
@@ -474,8 +471,8 @@ static int read_overlap_contain(struct pcache_cache_key *key, struct pcache_cach
 }
 
 /*
- *	 |-----------|		key_tmp (existing cached range)
- *	   |====|			key (requested range, fully within key_tmp)
+ * |-----------|		key_tmp (existing cached range)
+ *   |====|			key (requested range, fully within key_tmp)
  *
  * If `key_tmp` contains valid cached data, this function copies the relevant
  * portion to the request's bio. Otherwise, it sends a backing request to
@@ -524,8 +521,8 @@ static int read_overlap_contained(struct pcache_cache_key *key, struct pcache_ca
 }
 
 /*
- *	 |--------|		  key_tmp (existing cached range)
- *	   |==========|	  key (requested range, overlapping the head of key_tmp)
+ * |--------|		  key_tmp (existing cached range)
+ *   |==========|	  key (requested range, overlapping the head of key_tmp)
  */
 static int read_overlap_head(struct pcache_cache_key *key, struct pcache_cache_key *key_tmp,
 		struct pcache_cache_subtree_walk_ctx *ctx)

drivers/md/dm-pcache/cache_segment.c

@@ -243,8 +243,16 @@ struct pcache_cache_segment *get_cache_segment(struct pcache_cache *cache)
 
 	spin_lock(&cache->seg_map_lock);
 again:
-	seg_id = find_next_zero_bit(cache->seg_map, cache->n_segs, cache->last_cache_seg);
-	if (seg_id == cache->n_segs) {
+	/*
+	 * Only allocate initialized segments. cache_segs_init() initializes
+	 * cache_info.n_segs of the cache->n_segs device segments; a forged
+	 * smaller cache_info.n_segs leaves the rest as zeroed structs whose data
+	 * pointer is NULL. Bounding the search to cache_info.n_segs keeps such a
+	 * segment from reaching cache_kset_close(), which writes through it.
+	 */
+	seg_id = find_next_zero_bit(cache->seg_map, cache->cache_info.n_segs,
+				    cache->last_cache_seg);
+	if (seg_id == cache->cache_info.n_segs) {
 		/* reset the hint of ->last_cache_seg and retry */
 		if (cache->last_cache_seg) {
 			cache->last_cache_seg = 0;

drivers/md/dm-pcache/cache_writeback.c

@@ -48,18 +48,18 @@ static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cach
 	addr = cache_pos_addr(dirty_tail);
 	kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;
 
-	to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - dirty_tail->seg_off);
+	to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail));
 	ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy);
 	if (ret) {
 		pcache_dev_err(pcache, "error to read kset: %d", ret);
 		return true;
 	}
 
-	/* Check if the magic number matches the expected value */
-	if (kset_onmedia->magic != PCACHE_KSET_MAGIC) {
-		pcache_dev_debug(pcache, "dirty_tail: %u:%u magic: %llx, not expected: %llx\n",
+	/* Reject a corrupted or out-of-bounds kset before reading its keys */
+	if (!kset_onmedia_valid(kset_onmedia)) {
+		pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n",
 				dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off,
-				kset_onmedia->magic, PCACHE_KSET_MAGIC);
+				kset_onmedia->magic, kset_onmedia->key_num);
 		return true;
 	}
 
@@ -196,12 +196,18 @@ static int cache_kset_insert_tree(struct pcache_cache *cache, struct pcache_cach
 	return ret;
 }
 
-static void last_kset_writeback(struct pcache_cache *cache,
+static int last_kset_writeback(struct pcache_cache *cache,
 		struct pcache_cache_kset_onmedia *last_kset_onmedia)
 {
 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
 	struct pcache_cache_segment *next_seg;
 
+	if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) {
+		pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n",
+				last_kset_onmedia->next_cache_seg_id, cache->n_segs);
+		return -EIO;
+	}
+
 	pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id);
 
 	next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id];
@@ -211,6 +217,8 @@ static void last_kset_writeback(struct pcache_cache *cache,
 	cache->dirty_tail.seg_off = 0;
 	cache_encode_dirty_tail(cache);
 	mutex_unlock(&cache->dirty_tail_lock);
+
+	return 0;
 }
 
 void cache_writeback_fn(struct work_struct *work)
@@ -229,6 +237,9 @@ void cache_writeback_fn(struct work_struct *work)
 	if (pcache_is_stopping(pcache))
 		goto unlock;
 
+	if (atomic_read(&cache->writeback_errors))
+		goto unlock;
+
 	kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;
 
 	mutex_lock(&cache->dirty_tail_lock);
@@ -241,15 +252,24 @@ void cache_writeback_fn(struct work_struct *work)
 	}
 
 	if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) {
-		last_kset_writeback(cache, kset_onmedia);
+		ret = last_kset_writeback(cache, kset_onmedia);
+		if (ret) {
+			atomic_inc(&cache->writeback_errors);
+			goto unlock;
+		}
 		delay = 0;
 		goto queue_work;
 	}
 
+	if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) {
+		atomic_inc(&cache->writeback_errors);
+		goto unlock;
+	}
+
 	ret = cache_kset_insert_tree(cache, kset_onmedia);
 	if (ret) {
-		delay = PCACHE_CACHE_WRITEBACK_INTERVAL;
-		goto queue_work;
+		atomic_inc(&cache->writeback_errors);
+		goto unlock;
 	}
 
 	cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia));

drivers/md/dm-pcache/dm_pcache.c

@@ -439,13 +439,13 @@ static int dm_pcache_message(struct dm_target *ti, unsigned int argc,
 			     char **argv, char *result, unsigned int maxlen)
 {
 	struct dm_pcache *pcache = ti->private;
-	unsigned long val;
+	u8 val;
 
 	if (argc != 2)
 		goto err;
 
 	if (!strcasecmp(argv[0], "gc_percent")) {
-		if (kstrtoul(argv[1], 10, &val))
+		if (kstrtou8(argv[1], 10, &val))
 			goto err;
 
 		return pcache_cache_set_gc_percent(&pcache->cache, val);

drivers/md/dm-raid1.c

@@ -258,7 +258,7 @@ static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type)
 static int mirror_flush(struct dm_target *ti)
 {
 	struct mirror_set *ms = ti->private;
-	unsigned long error_bits;
+	unsigned long error_bits, unsup_bits;
 
 	unsigned int i;
 	struct dm_io_region io[MAX_NR_MIRRORS];
@@ -277,8 +277,8 @@ static int mirror_flush(struct dm_target *ti)
 	}
 
 	error_bits = -1;
-	dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT);
-	if (unlikely(error_bits != 0)) {
+	dm_io(&io_req, ms->nr_mirrors, io, &error_bits, &unsup_bits, IOPRIO_DEFAULT);
+	if (unlikely((error_bits | unsup_bits) != 0)) {
 		for (i = 0; i < ms->nr_mirrors; i++)
 			if (test_bit(i, &error_bits))
 				fail_mirror(ms->mirror + i,
@@ -511,7 +511,7 @@ static void hold_bio(struct mirror_set *ms, struct bio *bio)
  * Reads
  *---------------------------------------------------------------
  */
-static void read_callback(unsigned long error, void *context)
+static void read_callback(unsigned long error, unsigned long unsup, void *context)
 {
 	struct bio *bio = context;
 	struct mirror *m;
@@ -520,6 +520,8 @@ static void read_callback(unsigned long error, void *context)
 	bio_set_m(bio, NULL);
 
 	if (likely(!error)) {
+		if (unlikely(unsup != 0))
+			bio->bi_status = BLK_STS_INVAL;
 		bio_endio(bio);
 		return;
 	}
@@ -553,7 +555,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio)
 
 	map_region(&io, m, bio);
 	bio_set_m(bio, m);
-	BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT));
+	BUG_ON(dm_io(&io_req, 1, &io, NULL, NULL, IOPRIO_DEFAULT));
 }
 
 static inline int region_in_sync(struct mirror_set *ms, region_t region,
@@ -600,7 +602,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads)
  * NOSYNC:	increment pending, just write to the default mirror
  *---------------------------------------------------------------------
  */
-static void write_callback(unsigned long error, void *context)
+static void write_callback(unsigned long error, unsigned long unsup, void *context)
 {
 	unsigned int i;
 	struct bio *bio = context;
@@ -617,7 +619,7 @@ static void write_callback(unsigned long error, void *context)
 	 * This way we handle both writes to SYNC and NOSYNC
 	 * regions with the same code.
 	 */
-	if (likely(!error)) {
+	if (likely(!(error | unsup))) {
 		bio_endio(bio);
 		return;
 	}
@@ -632,6 +634,12 @@ static void write_callback(unsigned long error, void *context)
 		return;
 	}
 
+	if (!error && unsup) {
+		bio->bi_status = BLK_STS_INVAL;
+		bio_endio(bio);
+		return;
+	}
+
 	for (i = 0; i < ms->nr_mirrors; i++)
 		if (test_bit(i, &error))
 			fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR);
@@ -680,7 +688,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio)
 	 */
 	bio_set_m(bio, get_default_mirror(ms));
 
-	BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT));
+	BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, NULL, IOPRIO_DEFAULT));
 }
 
 static void do_writes(struct mirror_set *ms, struct bio_list *writes)
@@ -1262,7 +1270,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio,
 		return DM_ENDIO_DONE;
 	}
 
-	if (*error == BLK_STS_NOTSUPP)
+	if (*error == BLK_STS_NOTSUPP || *error == BLK_STS_INVAL)
 		goto out;
 
 	if (bio->bi_opf & REQ_RAHEAD)

drivers/md/dm-snap-persistent.c

@@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work)
 {
 	struct mdata_req *req = container_of(work, struct mdata_req, work);
 
-	req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT);
+	req->result = dm_io(req->io_req, 1, req->where, NULL, NULL, IOPRIO_DEFAULT);
 }
 
 /*
@@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf,
 	struct mdata_req req;
 
 	if (!metadata)
-		return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT);
+		return dm_io(&io_req, 1, &where, NULL, NULL, IOPRIO_DEFAULT);
 
 	req.where = &where;
 	req.io_req = &io_req;

drivers/md/dm-snap.c

@@ -911,9 +911,7 @@ static int init_hash_tables(struct dm_snapshot *s)
 
 static void merge_shutdown(struct dm_snapshot *s)
 {
-	clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
-	smp_mb__after_atomic();
-	wake_up_bit(&s->state_bits, RUNNING_MERGE);
+	clear_and_wake_up_bit(RUNNING_MERGE, &s->state_bits);
 }
 
 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)

drivers/md/dm-stats.c

@@ -178,8 +178,10 @@ static void dm_stat_free(struct rcu_head *head)
 	kfree(s->program_id);
 	kfree(s->aux_data);
 	for_each_possible_cpu(cpu) {
-		dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size);
-		dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);
+		if (s->stat_percpu[cpu]) {
+			dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size);
+			dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);
+		}
 	}
 	dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size);
 	dm_kvfree(s, s->shared_alloc_size);

drivers/md/dm-switch.c

@@ -184,7 +184,7 @@ static void switch_region_table_write(struct switch_ctx *sctx, unsigned long reg
 	pte = sctx->region_table[region_index];
 	pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit);
 	pte |= (region_table_slot_t)value << bit;
-	sctx->region_table[region_index] = pte;
+	WRITE_ONCE(sctx->region_table[region_index], pte);
 }
 
 /*

drivers/md/dm-table.c

@@ -2035,7 +2035,7 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
 		limits->features &= ~BLK_FEAT_NOWAIT;
 
 	/*
-	 * The current polling impementation does not support request based
+	 * The current polling implementation does not support request based
 	 * stacking.
 	 */
 	if (!__table_type_bio_based(t->type))

drivers/md/dm-vdo/indexer/config.c

@@ -29,7 +29,7 @@ static bool are_matching_configurations(struct uds_configuration *saved_config,
 					struct index_geometry *saved_geometry,
 					struct uds_configuration *user)
 {
-	struct index_geometry *geometry = user->geometry;
+	const struct index_geometry *geometry = &user->geometry;
 	bool result = true;
 
 	if (saved_geometry->record_pages_per_chapter != geometry->record_pages_per_chapter) {
@@ -141,8 +141,8 @@ int uds_validate_config_contents(struct buffered_reader *reader,
 		return UDS_CORRUPT_DATA;
 
 	if (is_version(INDEX_CONFIG_VERSION_6_02, version_buffer)) {
-		user_config->geometry->remapped_virtual = 0;
-		user_config->geometry->remapped_physical = 0;
+		user_config->geometry.remapped_virtual = 0;
+		user_config->geometry.remapped_physical = 0;
 	} else {
 		u8 remapping[sizeof(u64) + sizeof(u64)];
 
@@ -153,9 +153,9 @@ int uds_validate_config_contents(struct buffered_reader *reader,
 
 		offset = 0;
 		decode_u64_le(remapping, &offset,
-			      &user_config->geometry->remapped_virtual);
+			      &user_config->geometry.remapped_virtual);
 		decode_u64_le(remapping, &offset,
-			      &user_config->geometry->remapped_physical);
+			      &user_config->geometry.remapped_physical);
 	}
 
 	if (!are_matching_configurations(&config, &geometry, user_config)) {
@@ -175,7 +175,7 @@ int uds_write_config_contents(struct buffered_writer *writer,
 			      struct uds_configuration *config, u32 version)
 {
 	int result;
-	struct index_geometry *geometry = config->geometry;
+	const struct index_geometry *geometry = &config->geometry;
 	u8 buffer[sizeof(struct uds_configuration_8_02)];
 	size_t offset = 0;
 
@@ -329,13 +329,10 @@ int uds_make_configuration(const struct uds_parameters *params,
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = uds_make_index_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter,
-					 chapters_per_volume, sparse_chapters_per_volume,
-					 0, 0, &config->geometry);
-	if (result != UDS_SUCCESS) {
-		uds_free_configuration(config);
-		return result;
-	}
+	config->geometry =
+		uds_init_index_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter,
+					chapters_per_volume, sparse_chapters_per_volume,
+					0, 0);
 
 	config->zone_count = normalize_zone_count(params->zone_count);
 	config->read_threads = normalize_read_threads(params->read_threads);
@@ -355,22 +352,21 @@ int uds_make_configuration(const struct uds_parameters *params,
 void uds_free_configuration(struct uds_configuration *config)
 {
 	if (config != NULL) {
-		uds_free_index_geometry(config->geometry);
 		vdo_free(config);
 	}
 }
 
 void uds_log_configuration(struct uds_configuration *config)
 {
-	struct index_geometry *geometry = config->geometry;
+	const struct index_geometry geometry = config->geometry;
 
 	vdo_log_debug("Configuration:");
-	vdo_log_debug("  Record pages per chapter:   %10u", geometry->record_pages_per_chapter);
-	vdo_log_debug("  Chapters per volume:        %10u", geometry->chapters_per_volume);
-	vdo_log_debug("  Sparse chapters per volume: %10u", geometry->sparse_chapters_per_volume);
+	vdo_log_debug("  Record pages per chapter:   %10u", geometry.record_pages_per_chapter);
+	vdo_log_debug("  Chapters per volume:        %10u", geometry.chapters_per_volume);
+	vdo_log_debug("  Sparse chapters per volume: %10u", geometry.sparse_chapters_per_volume);
 	vdo_log_debug("  Cache size (chapters):      %10u", config->cache_chapters);
 	vdo_log_debug("  Volume index mean delta:    %10u", config->volume_index_mean_delta);
-	vdo_log_debug("  Bytes per page:             %10zu", geometry->bytes_per_page);
+	vdo_log_debug("  Bytes per page:             %10zu", geometry.bytes_per_page);
 	vdo_log_debug("  Sparse sample rate:         %10u", config->sparse_sample_rate);
 	vdo_log_debug("  Nonce:                      %llu", (unsigned long long) config->nonce);
 }

drivers/md/dm-vdo/indexer/config.h

@@ -37,7 +37,7 @@ struct uds_configuration {
 	/* Parameters for the volume */
 
 	/* The volume layout */
-	struct index_geometry *geometry;
+	struct index_geometry geometry;
 
 	/* Index owner's nonce */
 	u64 nonce;

drivers/md/dm-vdo/indexer/geometry.c

@@ -53,75 +53,51 @@
  * chapter it was moved to.
  */
 
-int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter,
-			    u32 chapters_per_volume, u32 sparse_chapters_per_volume,
-			    u64 remapped_virtual, u64 remapped_physical,
-			    struct index_geometry **geometry_ptr)
+struct index_geometry uds_init_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter,
+					      u32 chapters_per_volume, u32 sparse_chapters_per_volume,
+					      u64 remapped_virtual, u64 remapped_physical)
 {
-	int result;
-	struct index_geometry *geometry;
+	struct index_geometry geometry = {
+		.bytes_per_page = bytes_per_page,
+		.record_pages_per_chapter = record_pages_per_chapter,
+		.chapters_per_volume = chapters_per_volume,
+		.sparse_chapters_per_volume = sparse_chapters_per_volume,
+		.dense_chapters_per_volume = chapters_per_volume - sparse_chapters_per_volume,
+		.remapped_virtual = remapped_virtual,
+		.remapped_physical = remapped_physical,
+	};
 
-	result = vdo_allocate(1, "geometry", &geometry);
-	if (result != VDO_SUCCESS)
-		return result;
+	geometry.records_per_page = bytes_per_page / BYTES_PER_RECORD;
+	geometry.records_per_chapter = geometry.records_per_page * record_pages_per_chapter;
+	geometry.records_per_volume = (u64) geometry.records_per_chapter * chapters_per_volume;
 
-	geometry->bytes_per_page = bytes_per_page;
-	geometry->record_pages_per_chapter = record_pages_per_chapter;
-	geometry->chapters_per_volume = chapters_per_volume;
-	geometry->sparse_chapters_per_volume = sparse_chapters_per_volume;
-	geometry->dense_chapters_per_volume = chapters_per_volume - sparse_chapters_per_volume;
-	geometry->remapped_virtual = remapped_virtual;
-	geometry->remapped_physical = remapped_physical;
-
-	geometry->records_per_page = bytes_per_page / BYTES_PER_RECORD;
-	geometry->records_per_chapter = geometry->records_per_page * record_pages_per_chapter;
-	geometry->records_per_volume = (u64) geometry->records_per_chapter * chapters_per_volume;
-
-	geometry->chapter_mean_delta = 1 << DEFAULT_CHAPTER_MEAN_DELTA_BITS;
-	geometry->chapter_payload_bits = bits_per(record_pages_per_chapter - 1);
+	geometry.chapter_mean_delta = 1 << DEFAULT_CHAPTER_MEAN_DELTA_BITS;
+	geometry.chapter_payload_bits = bits_per(record_pages_per_chapter - 1);
 	/*
 	 * We want 1 delta list for every 64 records in the chapter.
 	 * The "| 077" ensures that the chapter_delta_list_bits computation
 	 * does not underflow.
 	 */
-	geometry->chapter_delta_list_bits =
-		bits_per((geometry->records_per_chapter - 1) | 077) - 6;
-	geometry->delta_lists_per_chapter = 1 << geometry->chapter_delta_list_bits;
+	geometry.chapter_delta_list_bits = bits_per((geometry.records_per_chapter - 1) | 077) - 6;
+	geometry.delta_lists_per_chapter = 1 << geometry.chapter_delta_list_bits;
 	/* We need enough address bits to achieve the desired mean delta. */
-	geometry->chapter_address_bits =
+	geometry.chapter_address_bits =
 		(DEFAULT_CHAPTER_MEAN_DELTA_BITS -
-		 geometry->chapter_delta_list_bits +
-		 bits_per(geometry->records_per_chapter - 1));
-	geometry->index_pages_per_chapter =
-		uds_get_delta_index_page_count(geometry->records_per_chapter,
-					       geometry->delta_lists_per_chapter,
-					       geometry->chapter_mean_delta,
-					       geometry->chapter_payload_bits,
+		 geometry.chapter_delta_list_bits +
+		 bits_per(geometry.records_per_chapter - 1));
+	geometry.index_pages_per_chapter =
+		uds_get_delta_index_page_count(geometry.records_per_chapter,
+					       geometry.delta_lists_per_chapter,
+					       geometry.chapter_mean_delta,
+					       geometry.chapter_payload_bits,
 					       bytes_per_page);
 
-	geometry->pages_per_chapter = geometry->index_pages_per_chapter + record_pages_per_chapter;
-	geometry->pages_per_volume = geometry->pages_per_chapter * chapters_per_volume;
-	geometry->bytes_per_volume =
-		bytes_per_page * (geometry->pages_per_volume + HEADER_PAGES_PER_VOLUME);
+	geometry.pages_per_chapter = geometry.index_pages_per_chapter + record_pages_per_chapter;
+	geometry.pages_per_volume = geometry.pages_per_chapter * chapters_per_volume;
+	geometry.bytes_per_volume =
+		bytes_per_page * (geometry.pages_per_volume + HEADER_PAGES_PER_VOLUME);
 
-	*geometry_ptr = geometry;
-	return UDS_SUCCESS;
-}
-
-int uds_copy_index_geometry(struct index_geometry *source,
-			    struct index_geometry **geometry_ptr)
-{
-	return uds_make_index_geometry(source->bytes_per_page,
-				       source->record_pages_per_chapter,
-				       source->chapters_per_volume,
-				       source->sparse_chapters_per_volume,
-				       source->remapped_virtual, source->remapped_physical,
-				       geometry_ptr);
-}
-
-void uds_free_index_geometry(struct index_geometry *geometry)
-{
-	vdo_free(geometry);
+	return geometry;
 }
 
 u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry,

drivers/md/dm-vdo/indexer/geometry.h

@@ -95,16 +95,9 @@ enum {
 	HEADER_PAGES_PER_VOLUME = 1,
 };
 
-int __must_check uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter,
-					 u32 chapters_per_volume,
-					 u32 sparse_chapters_per_volume, u64 remapped_virtual,
-					 u64 remapped_physical,
-					 struct index_geometry **geometry_ptr);
-
-int __must_check uds_copy_index_geometry(struct index_geometry *source,
-					 struct index_geometry **geometry_ptr);
-
-void uds_free_index_geometry(struct index_geometry *geometry);
+struct index_geometry uds_init_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter,
+					      u32 chapters_per_volume, u32 sparse_chapters_per_volume,
+					      u64 remapped_virtual, u64 remapped_physical);
 
 u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry,
 					     u64 virtual_chapter);

drivers/md/dm-vdo/indexer/index-layout.c

@@ -222,7 +222,7 @@ static int __must_check compute_sizes(const struct uds_configuration *config,
 				      struct save_layout_sizes *sls)
 {
 	int result;
-	struct index_geometry *geometry = config->geometry;
+	const struct index_geometry *geometry = &config->geometry;
 
 	memset(sls, 0, sizeof(*sls));
 	sls->save_count = MAX_SAVES;

drivers/md/dm-vdo/indexer/index.c

@@ -77,7 +77,7 @@ struct chapter_writer {
 
 static bool is_zone_chapter_sparse(const struct index_zone *zone, u64 virtual_chapter)
 {
-	return uds_is_chapter_sparse(zone->index->volume->geometry,
+	return uds_is_chapter_sparse(&zone->index->volume->geometry,
 				     zone->oldest_virtual_chapter,
 				     zone->newest_virtual_chapter, virtual_chapter);
 }
@@ -154,7 +154,7 @@ static int simulate_index_zone_barrier_message(struct index_zone *zone,
 	u64 sparse_virtual_chapter;
 
 	if ((zone->index->zone_count > 1) ||
-	    !uds_is_sparse_index_geometry(zone->index->volume->geometry))
+	    !uds_is_sparse_index_geometry(&zone->index->volume->geometry))
 		return UDS_SUCCESS;
 
 	sparse_virtual_chapter = triage_index_request(zone->index, request);
@@ -278,7 +278,7 @@ static int open_next_chapter(struct index_zone *zone)
 	}
 
 	expiring = zone->oldest_virtual_chapter;
-	expire_chapters = uds_chapters_to_expire(zone->index->volume->geometry,
+	expire_chapters = uds_chapters_to_expire(&zone->index->volume->geometry,
 						 zone->newest_virtual_chapter);
 	zone->oldest_virtual_chapter += expire_chapters;
 
@@ -353,7 +353,7 @@ static int search_sparse_cache_in_zone(struct index_zone *zone, struct uds_reque
 
 	request->virtual_chapter = virtual_chapter;
 	volume = zone->index->volume;
-	chapter = uds_map_to_physical_chapter(volume->geometry, virtual_chapter);
+	chapter = uds_map_to_physical_chapter(&volume->geometry, virtual_chapter);
 	return uds_search_cached_record_page(volume, request, chapter,
 					     record_page_number, found);
 }
@@ -470,7 +470,7 @@ static int search_index_zone(struct index_zone *zone, struct uds_request *reques
 			found = true;
 		} else if (request->location == UDS_LOCATION_UNAVAILABLE) {
 			found = false;
-		} else if (uds_is_sparse_index_geometry(zone->index->volume->geometry) &&
+		} else if (uds_is_sparse_index_geometry(&zone->index->volume->geometry) &&
 			   !uds_is_volume_index_sample(zone->index->volume_index,
 						       &request->record_name)) {
 			result = search_sparse_cache_in_zone(zone, request, NO_CHAPTER,
@@ -720,7 +720,7 @@ static void close_chapters(void *arg)
 		mutex_lock(&writer->mutex);
 		index->newest_virtual_chapter++;
 		index->oldest_virtual_chapter +=
-			uds_chapters_to_expire(index->volume->geometry,
+			uds_chapters_to_expire(&index->volume->geometry,
 					       index->newest_virtual_chapter);
 		writer->result = result;
 		writer->zones_to_write = 0;
@@ -762,7 +762,7 @@ static int make_chapter_writer(struct uds_index *index,
 	int result;
 	struct chapter_writer *writer;
 	size_t collated_records_size =
-		(sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter);
+		(sizeof(struct uds_volume_record) * index->volume->geometry.records_per_chapter);
 
 	result = vdo_allocate_extended(index->zone_count, chapters, "Chapter Writer", &writer);
 	if (result != VDO_SUCCESS)
@@ -780,7 +780,7 @@ static int make_chapter_writer(struct uds_index *index,
 	}
 
 	result = uds_make_open_chapter_index(&writer->open_chapter_index,
-					     index->volume->geometry,
+					     &index->volume->geometry,
 					     index->volume->nonce);
 	if (result != UDS_SUCCESS) {
 		free_chapter_writer(writer);
@@ -824,7 +824,7 @@ static int rebuild_index_page_map(struct uds_index *index, u64 vcn)
 {
 	int result;
 	struct delta_index_page *chapter_index_page;
-	struct index_geometry *geometry = index->volume->geometry;
+	struct index_geometry *geometry = &index->volume->geometry;
 	u32 chapter = uds_map_to_physical_chapter(geometry, vcn);
 	u32 expected_list_number = 0;
 	u32 index_page_number;
@@ -980,7 +980,7 @@ static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse)
 		return -EBUSY;
 	}
 
-	geometry = index->volume->geometry;
+	geometry = &index->volume->geometry;
 	physical_chapter = uds_map_to_physical_chapter(geometry, virtual);
 	uds_prefetch_volume_chapter(index->volume, physical_chapter);
 	uds_set_volume_index_open_chapter(index->volume_index, virtual);
@@ -1046,7 +1046,7 @@ static int replay_volume(struct uds_index *index)
 	 */
 	old_map_update = index->volume->index_page_map->last_update;
 	for (virtual = from_virtual; virtual < upto_virtual; virtual++) {
-		will_be_sparse = uds_is_chapter_sparse(index->volume->geometry,
+		will_be_sparse = uds_is_chapter_sparse(&index->volume->geometry,
 						       from_virtual, upto_virtual,
 						       virtual);
 		result = replay_chapter(index, virtual, will_be_sparse);
@@ -1073,7 +1073,7 @@ static int rebuild_index(struct uds_index *index)
 	u64 lowest;
 	u64 highest;
 	bool is_empty = false;
-	u32 chapters_per_volume = index->volume->geometry->chapters_per_volume;
+	u32 chapters_per_volume = index->volume->geometry.chapters_per_volume;
 
 	index->volume->lookup_mode = LOOKUP_FOR_REBUILD;
 	result = uds_find_volume_chapter_boundaries(index->volume, &lowest, &highest,
@@ -1125,14 +1125,14 @@ static int make_index_zone(struct uds_index *index, unsigned int zone_number)
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = uds_make_open_chapter(index->volume->geometry, index->zone_count,
+	result = uds_make_open_chapter(&index->volume->geometry, index->zone_count,
 				       &zone->open_chapter);
 	if (result != UDS_SUCCESS) {
 		free_index_zone(zone);
 		return result;
 	}
 
-	result = uds_make_open_chapter(index->volume->geometry, index->zone_count,
+	result = uds_make_open_chapter(&index->volume->geometry, index->zone_count,
 				       &zone->writing_chapter);
 	if (result != UDS_SUCCESS) {
 		free_index_zone(zone);
@@ -1202,7 +1202,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op
 	index->load_context = load_context;
 	index->callback = callback;
 
-	result = initialize_index_queues(index, config->geometry);
+	result = initialize_index_queues(index, &config->geometry);
 	if (result != UDS_SUCCESS) {
 		uds_free_index(index);
 		return result;

drivers/md/dm-vdo/indexer/open-chapter.c

@@ -343,7 +343,7 @@ int uds_save_open_chapter(struct uds_index *index, struct buffered_writer *write
 	return uds_flush_buffered_writer(writer);
 }
 
-u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry)
+u64 uds_compute_saved_open_chapter_size(const struct index_geometry *geometry)
 {
 	unsigned int records_per_chapter = geometry->records_per_chapter;
 

drivers/md/dm-vdo/indexer/open-chapter.h

@@ -74,6 +74,6 @@ int __must_check uds_save_open_chapter(struct uds_index *index,
 int __must_check uds_load_open_chapter(struct uds_index *index,
 				       struct buffered_reader *reader);
 
-u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry);
+u64 uds_compute_saved_open_chapter_size(const struct index_geometry *geometry);
 
 #endif /* UDS_OPEN_CHAPTER_H */

drivers/md/dm-vdo/indexer/volume-index.c

@@ -60,6 +60,12 @@
  * the index.
  */
 
+enum sub_index_parameters_slot {
+	DENSE = 0,
+	HOOK = 0,
+	NON_HOOK = 1,
+};
+
 struct sub_index_parameters {
 	/* The number of bits in address mask */
 	u8 address_bits;
@@ -77,16 +83,10 @@ struct sub_index_parameters {
 	size_t memory_size;
 	/* The number of bytes the index should keep free at all times */
 	size_t target_free_bytes;
-};
-
-struct split_config {
-	/* The hook subindex configuration */
-	struct uds_configuration hook_config;
-	struct index_geometry hook_geometry;
-
-	/* The non-hook subindex configuration */
-	struct uds_configuration non_hook_config;
-	struct index_geometry non_hook_geometry;
+	/* The mean delta for the volume index */
+	u32 volume_index_mean_delta;
+	/* The number of threads used to process index requests */
+	unsigned int zone_count;
 };
 
 struct chapter_range {
@@ -196,8 +196,11 @@ unsigned int uds_get_volume_index_zone(const struct volume_index *volume_index,
 
 #define DELTA_LIST_SIZE 256
 
-static int compute_volume_sub_index_parameters(const struct uds_configuration *config,
-					       struct sub_index_parameters *params)
+static int compute_sub_index_parameters(const struct uds_configuration *config,
+					u64 records_per_chapter,
+					u32 chapters_per_volume,
+					bool reduced,
+					struct sub_index_parameters *params)
 {
 	u64 entries_in_volume_index, address_span;
 	u32 chapters_in_volume_index, invalid_chapters;
@@ -207,17 +210,15 @@ static int compute_volume_sub_index_parameters(const struct uds_configuration *c
 	u64 index_size_in_bits;
 	size_t expected_index_size;
 	u64 min_delta_lists = MAX_ZONES * MAX_ZONES;
-	struct index_geometry *geometry = config->geometry;
-	u64 records_per_chapter = geometry->records_per_chapter;
 
-	params->chapter_count = geometry->chapters_per_volume;
+	params->chapter_count = chapters_per_volume;
 	/*
 	 * Make sure that the number of delta list records in the volume index does not change when
 	 * the volume is reduced by one chapter. This preserves the mapping from name to volume
 	 * index delta list.
 	 */
 	rounded_chapters = params->chapter_count;
-	if (uds_is_reduced_index_geometry(geometry))
+	if (reduced)
 		rounded_chapters += 1;
 	delta_list_records = records_per_chapter * rounded_chapters;
 	address_count = config->volume_index_mean_delta * DELTA_LIST_SIZE;
@@ -274,9 +275,46 @@ static int compute_volume_sub_index_parameters(const struct uds_configuration *c
 	params->memory_size = expected_index_size * 106 / 100;
 
 	params->target_free_bytes = expected_index_size / 20;
+	params->volume_index_mean_delta = config->volume_index_mean_delta;
+	params->zone_count = config->zone_count;
 	return UDS_SUCCESS;
 }
 
+static int compute_volume_sub_index_parameters(const struct uds_configuration *config,
+					       struct sub_index_parameters *params)
+{
+	const struct index_geometry *geometry = &config->geometry;
+	u64 sample_records;
+	u64 dense_chapters;
+	int result;
+	bool reduced = uds_is_reduced_index_geometry(geometry);
+
+	if (!uds_is_sparse_index_geometry(&config->geometry)) {
+		return compute_sub_index_parameters(config,
+						    geometry->records_per_chapter,
+						    geometry->chapters_per_volume,
+						    reduced,
+						    &params[DENSE]);
+	}
+
+	dense_chapters = geometry->chapters_per_volume - geometry->sparse_chapters_per_volume;
+	sample_records = geometry->records_per_chapter / config->sparse_sample_rate;
+
+	result = compute_sub_index_parameters(config,
+					      sample_records,
+					      geometry->chapters_per_volume,
+					      reduced,
+					      &params[HOOK]);
+	if (result != UDS_SUCCESS)
+		return result;
+
+	return compute_sub_index_parameters(config,
+					    geometry->records_per_chapter - sample_records,
+					    dense_chapters,
+					    reduced,
+					    &params[NON_HOOK]);
+}
+
 static void uninitialize_volume_sub_index(struct volume_sub_index *sub_index)
 {
 	vdo_free(vdo_forget(sub_index->flush_chapters));
@@ -298,73 +336,32 @@ void uds_free_volume_index(struct volume_index *volume_index)
 }
 
 
-static int compute_volume_sub_index_save_bytes(const struct uds_configuration *config,
-					       size_t *bytes)
+static size_t compute_volume_sub_index_save_bytes(struct sub_index_parameters *params)
 {
-	struct sub_index_parameters params = { .address_bits = 0 };
-	int result;
-
-	result = compute_volume_sub_index_parameters(config, &params);
-	if (result != UDS_SUCCESS)
-		return result;
-
-	*bytes = (sizeof(struct sub_index_data) + params.list_count * sizeof(u64) +
-		  uds_compute_delta_index_save_bytes(params.list_count,
-						     params.memory_size));
-	return UDS_SUCCESS;
-}
-
-/* This function is only useful if the configuration includes sparse chapters. */
-static void split_configuration(const struct uds_configuration *config,
-				struct split_config *split)
-{
-	u64 sample_rate, sample_records;
-	u64 dense_chapters, sparse_chapters;
-
-	/* Start with copies of the base configuration. */
-	split->hook_config = *config;
-	split->hook_geometry = *config->geometry;
-	split->hook_config.geometry = &split->hook_geometry;
-	split->non_hook_config = *config;
-	split->non_hook_geometry = *config->geometry;
-	split->non_hook_config.geometry = &split->non_hook_geometry;
-
-	sample_rate = config->sparse_sample_rate;
-	sparse_chapters = config->geometry->sparse_chapters_per_volume;
-	dense_chapters = config->geometry->chapters_per_volume - sparse_chapters;
-	sample_records = config->geometry->records_per_chapter / sample_rate;
-
-	/* Adjust the number of records indexed for each chapter. */
-	split->hook_geometry.records_per_chapter = sample_records;
-	split->non_hook_geometry.records_per_chapter -= sample_records;
-
-	/* Adjust the number of chapters indexed. */
-	split->hook_geometry.sparse_chapters_per_volume = 0;
-	split->non_hook_geometry.sparse_chapters_per_volume = 0;
-	split->non_hook_geometry.chapters_per_volume = dense_chapters;
+	return (sizeof(struct sub_index_data) + params->list_count * sizeof(u64) +
+		uds_compute_delta_index_save_bytes(params->list_count,
+						   params->memory_size));
 }
 
 static int compute_volume_index_save_bytes(const struct uds_configuration *config,
 					   size_t *bytes)
 {
-	size_t hook_bytes, non_hook_bytes;
-	struct split_config split;
 	int result;
+	struct sub_index_parameters parameters[2] = {
+		{ .address_bits = 0 },
+		{ .address_bits = 0 },
+	};
 
-	if (!uds_is_sparse_index_geometry(config->geometry))
-		return compute_volume_sub_index_save_bytes(config, bytes);
-
-	split_configuration(config, &split);
-	result = compute_volume_sub_index_save_bytes(&split.hook_config, &hook_bytes);
+	result = compute_volume_sub_index_parameters(config, parameters);
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = compute_volume_sub_index_save_bytes(&split.non_hook_config,
-						     &non_hook_bytes);
-	if (result != UDS_SUCCESS)
-		return result;
+	*bytes = compute_volume_sub_index_save_bytes(&parameters[HOOK]);
+	if (uds_is_sparse_index_geometry(&config->geometry)) {
+		*bytes += compute_volume_sub_index_save_bytes(&parameters[NON_HOOK]);
+		*bytes += sizeof(struct volume_index_data);
+	}
 
-	*bytes = sizeof(struct volume_index_data) + hook_bytes + non_hook_bytes;
 	return UDS_SUCCESS;
 }
 
@@ -1170,48 +1167,43 @@ void uds_get_volume_index_stats(const struct volume_index *volume_index,
 	stats->early_flushes += sparse_stats.early_flushes;
 }
 
-static int initialize_volume_sub_index(const struct uds_configuration *config,
+static int initialize_volume_sub_index(struct sub_index_parameters *params,
 				       u64 volume_nonce, u8 tag,
 				       struct volume_sub_index *sub_index)
 {
-	struct sub_index_parameters params = { .address_bits = 0 };
-	unsigned int zone_count = config->zone_count;
+	unsigned int zone_count = params->zone_count;
 	u64 available_bytes = 0;
 	unsigned int z;
 	int result;
 
-	result = compute_volume_sub_index_parameters(config, &params);
-	if (result != UDS_SUCCESS)
-		return result;
-
-	sub_index->address_bits = params.address_bits;
-	sub_index->address_mask = (1u << params.address_bits) - 1;
-	sub_index->chapter_bits = params.chapter_bits;
-	sub_index->chapter_mask = (1u << params.chapter_bits) - 1;
-	sub_index->chapter_count = params.chapter_count;
-	sub_index->list_count = params.list_count;
+	sub_index->address_bits = params->address_bits;
+	sub_index->address_mask = (1u << params->address_bits) - 1;
+	sub_index->chapter_bits = params->chapter_bits;
+	sub_index->chapter_mask = (1u << params->chapter_bits) - 1;
+	sub_index->chapter_count = params->chapter_count;
+	sub_index->list_count = params->list_count;
 	sub_index->zone_count = zone_count;
-	sub_index->chapter_zone_bits = params.chapter_size_in_bits / zone_count;
+	sub_index->chapter_zone_bits = params->chapter_size_in_bits / zone_count;
 	sub_index->volume_nonce = volume_nonce;
 
 	result = uds_initialize_delta_index(&sub_index->delta_index, zone_count,
-					    params.list_count, params.mean_delta,
-					    params.chapter_bits, params.memory_size,
+					    params->list_count, params->mean_delta,
+					    params->chapter_bits, params->memory_size,
 					    tag);
 	if (result != UDS_SUCCESS)
 		return result;
 
 	for (z = 0; z < sub_index->delta_index.zone_count; z++)
 		available_bytes += sub_index->delta_index.delta_zones[z].size;
-	available_bytes -= params.target_free_bytes;
+	available_bytes -= params->target_free_bytes;
 	sub_index->max_zone_bits = (available_bytes * BITS_PER_BYTE) / zone_count;
 	sub_index->memory_size = (sub_index->delta_index.memory_size +
 				  sizeof(struct volume_sub_index) +
-				  (params.list_count * sizeof(u64)) +
+				  (params->list_count * sizeof(u64)) +
 				  (zone_count * sizeof(struct volume_sub_index_zone)));
 
 	/* The following arrays are initialized to all zeros. */
-	result = vdo_allocate(params.list_count, "first chapter to flush",
+	result = vdo_allocate(params->list_count, "first chapter to flush",
 			      &sub_index->flush_chapters);
 	if (result != VDO_SUCCESS)
 		return result;
@@ -1222,10 +1214,13 @@ static int initialize_volume_sub_index(const struct uds_configuration *config,
 int uds_make_volume_index(const struct uds_configuration *config, u64 volume_nonce,
 			  struct volume_index **volume_index_ptr)
 {
-	struct split_config split;
 	unsigned int zone;
 	struct volume_index *volume_index;
 	int result;
+	struct sub_index_parameters parameters[2] = {
+		{ .address_bits = 0 },
+		{ .address_bits = 0 },
+	};
 
 	result = vdo_allocate(1, "volume index", &volume_index);
 	if (result != VDO_SUCCESS)
@@ -1233,8 +1228,12 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non
 
 	volume_index->zone_count = config->zone_count;
 
-	if (!uds_is_sparse_index_geometry(config->geometry)) {
-		result = initialize_volume_sub_index(config, volume_nonce, 'm',
+	result = compute_volume_sub_index_parameters(config, parameters);
+	if (result != UDS_SUCCESS)
+		return result;
+
+	if (!uds_is_sparse_index_geometry(&config->geometry)) {
+		result = initialize_volume_sub_index(&parameters[DENSE], volume_nonce, 'm',
 						     &volume_index->vi_non_hook);
 		if (result != UDS_SUCCESS) {
 			uds_free_volume_index(volume_index);
@@ -1257,8 +1256,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non
 	for (zone = 0; zone < config->zone_count; zone++)
 		mutex_init(&volume_index->zones[zone].hook_mutex);
 
-	split_configuration(config, &split);
-	result = initialize_volume_sub_index(&split.non_hook_config, volume_nonce, 'd',
+	result = initialize_volume_sub_index(&parameters[NON_HOOK], volume_nonce, 'd',
 					     &volume_index->vi_non_hook);
 	if (result != UDS_SUCCESS) {
 		uds_free_volume_index(volume_index);
@@ -1266,7 +1264,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non
 					      "Error creating non hook volume index");
 	}
 
-	result = initialize_volume_sub_index(&split.hook_config, volume_nonce, 's',
+	result = initialize_volume_sub_index(&parameters[HOOK], volume_nonce, 's',
 					     &volume_index->vi_hook);
 	if (result != UDS_SUCCESS) {
 		uds_free_volume_index(volume_index);

drivers/md/dm-vdo/indexer/volume.c

@@ -83,17 +83,17 @@ union invalidate_counter {
 	};
 };
 
-static inline u32 map_to_page_number(struct index_geometry *geometry, u32 physical_page)
+static inline u32 map_to_page_number(const struct index_geometry *geometry, u32 physical_page)
 {
 	return (physical_page - HEADER_PAGES_PER_VOLUME) % geometry->pages_per_chapter;
 }
 
-static inline u32 map_to_chapter_number(struct index_geometry *geometry, u32 physical_page)
+static inline u32 map_to_chapter_number(const struct index_geometry *geometry, u32 physical_page)
 {
 	return (physical_page - HEADER_PAGES_PER_VOLUME) / geometry->pages_per_chapter;
 }
 
-static inline bool is_record_page(struct index_geometry *geometry, u32 physical_page)
+static inline bool is_record_page(const struct index_geometry *geometry, u32 physical_page)
 {
 	return map_to_page_number(geometry, physical_page) >= geometry->index_pages_per_chapter;
 }
@@ -422,7 +422,7 @@ static int init_chapter_index_page(const struct volume *volume, u8 *index_page,
 	u32 ci_chapter;
 	u32 lowest_list;
 	u32 highest_list;
-	struct index_geometry *geometry = volume->geometry;
+	const struct index_geometry *geometry = &volume->geometry;
 	int result;
 
 	result = uds_initialize_chapter_index_page(chapter_index_page, geometry,
@@ -459,8 +459,8 @@ static int init_chapter_index_page(const struct volume *volume, u8 *index_page,
 static int initialize_index_page(const struct volume *volume, u32 physical_page,
 				 struct cached_page *page)
 {
-	u32 chapter = map_to_chapter_number(volume->geometry, physical_page);
-	u32 index_page_number = map_to_page_number(volume->geometry, physical_page);
+	u32 chapter = map_to_chapter_number(&volume->geometry, physical_page);
+	u32 index_page_number = map_to_page_number(&volume->geometry, physical_page);
 
 	return init_chapter_index_page(volume, dm_bufio_get_block_data(page->buffer),
 				       chapter, index_page_number, &page->index_page);
@@ -510,16 +510,16 @@ static int search_page(struct cached_page *page, const struct volume *volume,
 	enum uds_index_region location;
 	u16 record_page_number;
 
-	if (is_record_page(volume->geometry, physical_page)) {
+	if (is_record_page(&volume->geometry, physical_page)) {
 		if (search_record_page(dm_bufio_get_block_data(page->buffer),
-				       &request->record_name, volume->geometry,
+				       &request->record_name, &volume->geometry,
 				       &request->old_metadata))
 			location = UDS_LOCATION_RECORD_PAGE_LOOKUP;
 		else
 			location = UDS_LOCATION_UNAVAILABLE;
 	} else {
 		result = uds_search_chapter_index_page(&page->index_page,
-						       volume->geometry,
+						       &volume->geometry,
 						       &request->record_name,
 						       &record_page_number);
 		if (result != UDS_SUCCESS)
@@ -571,7 +571,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry)
 		return UDS_SUCCESS;
 	}
 
-	if (!is_record_page(volume->geometry, page_number)) {
+	if (!is_record_page(&volume->geometry, page_number)) {
 		result = initialize_index_page(volume, page_number, page);
 		if (result != UDS_SUCCESS) {
 			vdo_log_warning("Error initializing chapter index page");
@@ -708,7 +708,7 @@ static int read_page_locked(struct volume *volume, u32 physical_page,
 		return result;
 	}
 
-	if (!is_record_page(volume->geometry, physical_page)) {
+	if (!is_record_page(&volume->geometry, physical_page)) {
 		result = initialize_index_page(volume, physical_page, page);
 		if (result != UDS_SUCCESS) {
 			if (volume->lookup_mode != LOOKUP_FOR_REBUILD)
@@ -807,7 +807,7 @@ static int get_volume_page(struct volume *volume, u32 chapter, u32 page_number,
 			   struct cached_page **page_ptr)
 {
 	int result;
-	u32 physical_page = map_to_physical_page(volume->geometry, chapter, page_number);
+	u32 physical_page = map_to_physical_page(&volume->geometry, chapter, page_number);
 
 	mutex_lock(&volume->read_threads_mutex);
 	result = get_volume_page_locked(volume, physical_page, page_ptr);
@@ -850,7 +850,7 @@ static int search_cached_index_page(struct volume *volume, struct uds_request *r
 	int result;
 	struct cached_page *page = NULL;
 	unsigned int zone_number = request->zone_number;
-	u32 physical_page = map_to_physical_page(volume->geometry, chapter,
+	u32 physical_page = map_to_physical_page(&volume->geometry, chapter,
 						 index_page_number);
 
 	/*
@@ -867,7 +867,7 @@ static int search_cached_index_page(struct volume *volume, struct uds_request *r
 		return result;
 	}
 
-	result = uds_search_chapter_index_page(&page->index_page, volume->geometry,
+	result = uds_search_chapter_index_page(&page->index_page, &volume->geometry,
 					       &request->record_name,
 					       record_page_number);
 	end_pending_search(&volume->page_cache, zone_number);
@@ -882,7 +882,7 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req
 				  u32 chapter, u16 record_page_number, bool *found)
 {
 	struct cached_page *record_page;
-	struct index_geometry *geometry = volume->geometry;
+	const struct index_geometry *geometry = &volume->geometry;
 	unsigned int zone_number = request->zone_number;
 	int result;
 	u32 physical_page, page_number;
@@ -899,7 +899,7 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req
 
 	page_number = geometry->index_pages_per_chapter + record_page_number;
 
-	physical_page = map_to_physical_page(volume->geometry, chapter, page_number);
+	physical_page = map_to_physical_page(&volume->geometry, chapter, page_number);
 
 	/*
 	 * Make sure the invalidate counter is updated before we try and read the mapping. This
@@ -925,7 +925,7 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req
 
 void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter)
 {
-	const struct index_geometry *geometry = volume->geometry;
+	const struct index_geometry *geometry = &volume->geometry;
 	u32 physical_page = map_to_physical_page(geometry, chapter, 0);
 
 	dm_bufio_prefetch(volume->client, physical_page, geometry->pages_per_chapter);
@@ -937,7 +937,7 @@ int uds_read_chapter_index_from_volume(const struct volume *volume, u64 virtual_
 {
 	int result;
 	u32 i;
-	const struct index_geometry *geometry = volume->geometry;
+	const struct index_geometry *geometry = &volume->geometry;
 	u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter);
 	u32 physical_page = map_to_physical_page(geometry, physical_chapter, 0);
 
@@ -969,7 +969,7 @@ int uds_search_volume_page_cache(struct volume *volume, struct uds_request *requ
 {
 	int result;
 	u32 physical_chapter =
-		uds_map_to_physical_chapter(volume->geometry, request->virtual_chapter);
+		uds_map_to_physical_chapter(&volume->geometry, request->virtual_chapter);
 	u32 index_page_number;
 	u16 record_page_number;
 
@@ -996,7 +996,7 @@ int uds_search_volume_page_cache_for_rebuild(struct volume *volume,
 					     u64 virtual_chapter, bool *found)
 {
 	int result;
-	struct index_geometry *geometry = volume->geometry;
+	struct index_geometry *geometry = &volume->geometry;
 	struct cached_page *page;
 	u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter);
 	u32 index_page_number;
@@ -1049,13 +1049,13 @@ static void invalidate_page(struct page_cache *cache, u32 physical_page)
 void uds_forget_chapter(struct volume *volume, u64 virtual_chapter)
 {
 	u32 physical_chapter =
-		uds_map_to_physical_chapter(volume->geometry, virtual_chapter);
-	u32 first_page = map_to_physical_page(volume->geometry, physical_chapter, 0);
+		uds_map_to_physical_chapter(&volume->geometry, virtual_chapter);
+	u32 first_page = map_to_physical_page(&volume->geometry, physical_chapter, 0);
 	u32 i;
 
 	vdo_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter);
 	mutex_lock(&volume->read_threads_mutex);
-	for (i = 0; i < volume->geometry->pages_per_chapter; i++)
+	for (i = 0; i < volume->geometry.pages_per_chapter; i++)
 		invalidate_page(&volume->page_cache, first_page + i);
 	mutex_unlock(&volume->read_threads_mutex);
 }
@@ -1070,7 +1070,7 @@ static int donate_index_page_locked(struct volume *volume, u32 physical_chapter,
 	int result;
 	struct cached_page *page = NULL;
 	u32 physical_page =
-		map_to_physical_page(volume->geometry, physical_chapter,
+		map_to_physical_page(&volume->geometry, physical_chapter,
 				     index_page_number);
 
 	page = select_victim_in_cache(&volume->page_cache);
@@ -1097,7 +1097,7 @@ static int donate_index_page_locked(struct volume *volume, u32 physical_chapter,
 static int write_index_pages(struct volume *volume, u32 physical_chapter_number,
 			     struct open_chapter_index *chapter_index)
 {
-	struct index_geometry *geometry = volume->geometry;
+	struct index_geometry *geometry = &volume->geometry;
 	struct dm_buffer *page_buffer;
 	u32 first_index_page = map_to_physical_page(geometry, physical_chapter_number, 0);
 	u32 delta_list_number = 0;
@@ -1184,7 +1184,7 @@ static int encode_record_page(const struct volume *volume,
 {
 	int result;
 	u32 i;
-	u32 records_per_page = volume->geometry->records_per_page;
+	u32 records_per_page = volume->geometry.records_per_page;
 	const struct uds_volume_record **record_pointers = volume->record_pointers;
 
 	for (i = 0; i < records_per_page; i++)
@@ -1208,7 +1208,7 @@ static int write_record_pages(struct volume *volume, u32 physical_chapter_number
 			      const struct uds_volume_record *records)
 {
 	u32 record_page_number;
-	struct index_geometry *geometry = volume->geometry;
+	struct index_geometry *geometry = &volume->geometry;
 	struct dm_buffer *page_buffer;
 	const struct uds_volume_record *next_record = records;
 	u32 first_record_page = map_to_physical_page(geometry, physical_chapter_number,
@@ -1248,7 +1248,7 @@ int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_
 {
 	int result;
 	u32 physical_chapter_number =
-		uds_map_to_physical_chapter(volume->geometry,
+		uds_map_to_physical_chapter(&volume->geometry,
 					    chapter_index->virtual_chapter_number);
 
 	result = write_index_pages(volume, physical_chapter_number, chapter_index);
@@ -1269,7 +1269,7 @@ int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_
 static void probe_chapter(struct volume *volume, u32 chapter_number,
 			  u64 *virtual_chapter_number)
 {
-	const struct index_geometry *geometry = volume->geometry;
+	const struct index_geometry *geometry = &volume->geometry;
 	u32 expected_list_number = 0;
 	u32 i;
 	u64 vcn = BAD_CHAPTER;
@@ -1353,7 +1353,7 @@ static void find_real_end_of_volume(struct volume *volume, u32 limit, u32 *limit
 static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lowest_vcn,
 			       u64 *highest_vcn)
 {
-	struct index_geometry *geometry = volume->geometry;
+	struct index_geometry *geometry = &volume->geometry;
 	u64 zero_vcn;
 	u64 lowest = BAD_CHAPTER;
 	u64 highest = BAD_CHAPTER;
@@ -1451,7 +1451,7 @@ static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lo
 int uds_find_volume_chapter_boundaries(struct volume *volume, u64 *lowest_vcn,
 				       u64 *highest_vcn, bool *is_empty)
 {
-	u32 chapter_limit = volume->geometry->chapters_per_volume;
+	u32 chapter_limit = volume->geometry.chapters_per_volume;
 
 	find_real_end_of_volume(volume, chapter_limit, &chapter_limit);
 	if (chapter_limit == 0) {
@@ -1486,7 +1486,7 @@ int __must_check uds_replace_volume_storage(struct volume *volume,
 	if (volume->client != NULL)
 		dm_bufio_client_destroy(vdo_forget(volume->client));
 
-	return uds_open_volume_bufio(layout, volume->geometry->bytes_per_page,
+	return uds_open_volume_bufio(layout, volume->geometry.bytes_per_page,
 				     volume->reserved_buffers, &volume->client);
 }
 
@@ -1552,13 +1552,8 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout
 
 	volume->nonce = uds_get_volume_nonce(layout);
 
-	result = uds_copy_index_geometry(config->geometry, &volume->geometry);
-	if (result != UDS_SUCCESS) {
-		uds_free_volume(volume);
-		return vdo_log_warning_strerror(result,
-						"failed to allocate geometry: error");
-	}
-	geometry = volume->geometry;
+	volume->geometry = config->geometry;
+	geometry = &volume->geometry;
 
 	/*
 	 * Reserve a buffer for each entry in the page cache, one for the chapter writer, and one
@@ -1685,7 +1680,6 @@ void uds_free_volume(struct volume *volume)
 
 	uds_free_index_page_map(volume->index_page_map);
 	uds_free_radix_sorter(volume->radix_sorter);
-	vdo_free(volume->geometry);
 	vdo_free(volume->record_pointers);
 	vdo_free(volume);
 }

drivers/md/dm-vdo/indexer/volume.h

@@ -97,7 +97,7 @@ struct page_cache {
 };
 
 struct volume {
-	struct index_geometry *geometry;
+	struct index_geometry geometry;
 	struct dm_bufio_client *client;
 	u64 nonce;
 	size_t cache_size;

drivers/md/dm-vdo/repair.c

@@ -1696,6 +1696,7 @@ void vdo_repair(struct vdo_completion *parent)
 	struct vdo *vdo = parent->vdo;
 	struct recovery_journal *journal = vdo->recovery_journal;
 	physical_block_number_t pbn = journal->origin;
+	block_count_t i;
 	block_count_t remaining = journal->size;
 	block_count_t vio_count = DIV_ROUND_UP(remaining, MAX_BLOCKS_PER_VIO);
 	page_count_t page_count = min_t(page_count_t,
@@ -1749,9 +1750,8 @@ void vdo_repair(struct vdo_completion *parent)
 		remaining -= blocks;
 	}
 
-	for (vio_count = 0; vio_count < repair->vio_count;
-	     vio_count++, pbn += MAX_BLOCKS_PER_VIO) {
-		vdo_submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio,
+	for (i = 0; i < vio_count; i++, pbn += MAX_BLOCKS_PER_VIO) {
+		vdo_submit_metadata_vio(&repair->vios[i], pbn, read_journal_endio,
 					handle_journal_load_error, REQ_OP_READ);
 	}
 }

drivers/md/dm-verity-target.c

@@ -316,9 +316,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 		else if (verity_handle_err(v,
 					   DM_VERITY_BLOCK_TYPE_METADATA,
 					   hash_block)) {
-			struct bio *bio;
 			io->had_mismatch = true;
-			bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
 			dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio,
 					 block, 0);
 			r = -EIO;
@@ -395,7 +393,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
 	io_loc.bdev = v->data_dev->bdev;
 	io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
 	io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
-	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r))
 		goto free_ret;
 

drivers/md/dm-writecache.c

@@ -474,12 +474,14 @@ struct io_notify {
 	atomic_t count;
 };
 
-static void writecache_notify_io(unsigned long error, void *context)
+static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context)
 {
 	struct io_notify *endio = context;
 
 	if (unlikely(error != 0))
 		writecache_error(endio->wc, -EIO, "error writing metadata");
+	else if (unlikely(unsup != 0))
+		writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata");
 	BUG_ON(atomic_read(&endio->count) <= 0);
 	if (atomic_dec_and_test(&endio->count))
 		complete(&endio->c);
@@ -530,11 +532,11 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
 		req.notify.context = &endio;
 
 		/* writing via async dm-io (implied by notify.fn above) won't return an error */
-		(void) dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
+		(void) dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
 		i = j;
 	}
 
-	writecache_notify_io(0, &endio);
+	writecache_notify_io(0, 0, &endio);
 	wait_for_completion_io(&endio.c);
 
 	if (wait_for_ios)
@@ -567,7 +569,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc)
 	req.notify.fn = NULL;
 	req.notify.context = NULL;
 
-	r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r))
 		writecache_error(wc, r, "error writing superblock");
 }
@@ -595,7 +597,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
 	req.client = wc->dm_io;
 	req.notify.fn = NULL;
 
-	r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
+	r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
 	if (unlikely(r))
 		writecache_error(wc, r, "error flushing metadata: %d", r);
 }
@@ -989,7 +991,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors
 	req.client = wc->dm_io;
 	req.notify.fn = NULL;
 
-	return dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
+	return dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
 }
 
 static void writecache_resume(struct dm_target *ti)

drivers/md/dm-zoned-metadata.c

@@ -519,9 +519,7 @@ static void dmz_mblock_bio_end_io(struct bio *bio)
 	else
 		flag = DMZ_META_READING;
 
-	clear_bit_unlock(flag, &mblk->state);
-	smp_mb__after_atomic();
-	wake_up_bit(&mblk->state, flag);
+	clear_and_wake_up_bit(flag, &mblk->state);
 
 	bio_put(bio);
 }
@@ -1910,9 +1908,7 @@ void dmz_unlock_zone_reclaim(struct dm_zone *zone)
 	WARN_ON(dmz_is_active(zone));
 	WARN_ON(!dmz_in_reclaim(zone));
 
-	clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
-	smp_mb__after_atomic();
-	wake_up_bit(&zone->flags, DMZ_RECLAIM);
+	clear_and_wake_up_bit(DMZ_RECLAIM, &zone->flags);
 }
 
 /*

drivers/md/dm-zoned-reclaim.c

@@ -106,9 +106,7 @@ static void dmz_reclaim_kcopy_end(int read_err, unsigned long write_err,
 	else
 		zrc->kc_err = 0;
 
-	clear_bit_unlock(DMZ_RECLAIM_KCOPY, &zrc->flags);
-	smp_mb__after_atomic();
-	wake_up_bit(&zrc->flags, DMZ_RECLAIM_KCOPY);
+	clear_and_wake_up_bit(DMZ_RECLAIM_KCOPY, &zrc->flags);
 }
 
 /*

drivers/md/dm.c

@@ -2630,9 +2630,10 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
 	 */
 	mutex_lock(&md->table_devices_lock);
 	r = add_disk(md->disk);
-	mutex_unlock(&md->table_devices_lock);
-	if (r)
+	if (r) {
+		mutex_unlock(&md->table_devices_lock);
 		return r;
+	}
 
 	/*
 	 * Register the holder relationship for devices added before the disk
@@ -2643,18 +2644,21 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
 		if (r)
 			goto out_undo_holders;
 	}
+	mutex_unlock(&md->table_devices_lock);
 
 	r = dm_sysfs_init(md);
 	if (r)
-		goto out_undo_holders;
+		goto lock_out_undo_holders;
 
 	md->type = type;
+
 	return 0;
 
+lock_out_undo_holders:
+	mutex_lock(&md->table_devices_lock);
 out_undo_holders:
 	list_for_each_entry_continue_reverse(td, &md->table_devices, list)
 		bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
-	mutex_lock(&md->table_devices_lock);
 	del_gendisk(md->disk);
 	mutex_unlock(&md->table_devices_lock);
 	return r;
@@ -3140,7 +3144,7 @@ int dm_resume(struct mapped_device *md)
 	r = -EINVAL;
 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
 
-	if (!dm_suspended_md(md))
+	if (!dm_suspended_md(md) || test_bit(DMF_FREEING, &md->flags))
 		goto out;
 
 	if (dm_suspended_internally_md(md)) {

drivers/md/persistent-data/dm-array.c

@@ -38,6 +38,14 @@ struct array_block {
  */
 #define CSUM_XOR 595846735
 
+/*
+ * Each array block can hold this many values.
+ */
+static uint32_t calc_max_entries(size_t value_size, size_t size_of_block)
+{
+	return (size_of_block - sizeof(struct array_block)) / value_size;
+}
+
 static void array_block_prepare_for_write(const struct dm_block_validator *v,
 					  struct dm_block *b,
 					  size_t size_of_block)
@@ -55,6 +63,7 @@ static int array_block_check(const struct dm_block_validator *v,
 			     size_t size_of_block)
 {
 	struct array_block *bh_le = dm_block_data(b);
+	uint32_t nr_entries, max_entries, value_size;
 	__le32 csum_disk;
 
 	if (dm_block_location(b) != le64_to_cpu(bh_le->blocknr)) {
@@ -74,6 +83,26 @@ static int array_block_check(const struct dm_block_validator *v,
 		return -EILSEQ;
 	}
 
+	nr_entries = le32_to_cpu(bh_le->nr_entries);
+	max_entries = le32_to_cpu(bh_le->max_entries);
+	value_size = le32_to_cpu(bh_le->value_size);
+
+	if (!value_size) {
+		DMERR_LIMIT("%s failed: value_size is zero", __func__);
+		return -EILSEQ;
+	}
+
+	if (max_entries != calc_max_entries(value_size, size_of_block)) {
+		DMERR_LIMIT("%s failed: max_entries %u invalid for value_size %u",
+			    __func__, max_entries, value_size);
+		return -EILSEQ;
+	}
+
+	if (nr_entries > max_entries) {
+		DMERR_LIMIT("%s failed: too many entries", __func__);
+		return -EILSEQ;
+	}
+
 	return 0;
 }
 
@@ -138,14 +167,6 @@ static void dec_ablock_entries(struct dm_array_info *info, struct array_block *a
 		on_entries(info, ab, vt->dec);
 }
 
-/*
- * Each array block can hold this many values.
- */
-static uint32_t calc_max_entries(size_t value_size, size_t size_of_block)
-{
-	return (size_of_block - sizeof(struct array_block)) / value_size;
-}
-
 /*
  * Allocate a new array block.  The caller will need to unlock block.
  */
@@ -225,6 +246,14 @@ static int get_ablock(struct dm_array_info *info, dm_block_t b,
 		return r;
 
 	*ab = dm_block_data(*block);
+	if (le32_to_cpu((*ab)->value_size) != info->value_type.size) {
+		DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__,
+			    le32_to_cpu((*ab)->value_size),
+			    info->value_type.size);
+		dm_tm_unlock(info->btree_info.tm, *block);
+		return -EILSEQ;
+	}
+
 	return 0;
 }
 
@@ -287,6 +316,14 @@ static int __shadow_ablock(struct dm_array_info *info, dm_block_t b,
 		return r;
 
 	*ab = dm_block_data(*block);
+	if (le32_to_cpu((*ab)->value_size) != info->value_type.size) {
+		DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__,
+			    le32_to_cpu((*ab)->value_size),
+			    info->value_type.size);
+		dm_tm_unlock(info->btree_info.tm, *block);
+		return -EILSEQ;
+	}
+
 	if (inc)
 		inc_ablock_entries(info, *ab);
 

include/linux/dm-io.h

@@ -27,7 +27,7 @@ struct page_list {
 	struct page *page;
 };
 
-typedef void (*io_notify_fn)(unsigned int long error, void *context);
+typedef void (*io_notify_fn)(unsigned long int error, unsigned long int unsup, void *context);
 
 enum dm_io_mem_type {
 	DM_IO_PAGE_LIST,/* Page list */
@@ -80,8 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client);
  * error occurred doing io to the corresponding region.
  */
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-	  struct dm_io_region *region, unsigned int long *sync_error_bits,
-	  unsigned short ioprio);
+	  struct dm_io_region *region, unsigned long int *sync_error_bits,
+	  unsigned long int *sync_unsup_bits, unsigned short ioprio);
 
 #endif	/* __KERNEL__ */
 #endif	/* _LINUX_DM_IO_H */