github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

vfs-7.2-rc1.iomap

Browse Source 
Please consider pulling these changes from the signed vfs-7.2-rc1.iomap tag.
 
 Thanks!
 Christian
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQRAhzRXHqcMeLMyaSiRxhvAZXjcogUCaiwLKgAKCRCRxhvAZXjc
 os7sAPsH6z7HATCG9xOAOmsIwgi7l0u7dMNnU7Xrqs9H3qAjNgD/SJ/Ajw0GMkMP
 RxJvze9lvkY9hN9avR9thMVF+yE+fgM=
 =TlkI
 -----END PGP SIGNATURE-----

Merge tag 'vfs-7.2-rc1.iomap' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs

Pull iomap updates from Christian Brauner:

 - Add the vfs infrastructure required to implement fs-verity support
   for XFS with a post-EOF merkle tree: fsverity generates and stores a
   zero-block hash, and iomap learns to verify data on buffered reads,
   to handle fsverity during writeback via the new IOMAP_F_FSVERITY
   flag, and to write fsverity metadata through iomap_fsverity_write().

 - Skip the memset of the iomap in iomap_iter() once the iteration is
   done. In high-IOPS scenarios (4k randread NVMe polling via io_uring)
   the pointless memset wasted memory write bandwidth; this improves
   IOPS by about 5% on ext4 and xfs.

 - Add balance_dirty_pages_ratelimited() to iomap_zero_iter(), aligning
   it with iomap_write_iter(). This prepares for the exFAT iomap
   conversion where zeroing beyond valid_size can trigger large-scale
   zeroing operations that caused memory pressure without throttling.

 - Remove the over-strict inline data boundary check. If a filesystem
   provides a valid inline_data pointer and length there is no reason to
   require that inline data must not cross a page boundary.

 - Don't make REQ_POLLED imply REQ_NOWAIT, matching the earlier
   equivalent block layer fix: there are valid cases to poll for I/O
   completion without REQ_NOWAIT, and REQ_NOWAIT for file system writes
   is currently not supported as writes aren't idempotent.

 - Introduce IOMAP_F_ZERO_TAIL for filesystems that maintain a separate
   valid data length (exFAT, NTFS). For a write starting at or beyond
   valid_size, __iomap_write_begin() now zeroes only the tail portion of
   the block while preserving valid data before it, instead of leaving
   stale data in the page cache. The flag is also added to the iomap
   trace event strings.

* tag 'vfs-7.2-rc1.iomap' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs:
  iomap: Add IOMAP_F_ZERO_TAIL flag to trace event strings
  iomap: introduce iomap_fsverity_write() for writing fsverity metadata
  iomap: teach iomap to read files with fsverity
  iomap: introduce IOMAP_F_FSVERITY and teach writeback to handle fsverity
  fsverity: generate and store zero-block hash
  iomap: introduce IOMAP_F_ZERO_TAIL flag
  iomap: don't make REQ_POLLED imply REQ_NOWAIT
  iomap: remove over-strict inline data boundary check
  iomap: add dirty page control to iomap_zero_iter
  iomap: avoid memset iomap when iter is done
This commit is contained in:
Linus Torvalds 2026-06-15 03:46:54 +05:30
commit ec5d1ae94e
12 changed files with 170 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
fs/iomap/buffered-io.c
View File

@ -9,6 +9,7 @@
#include <linux/swap.h>
#include <linux/migrate.h>
#include <linux/fserror.h>
#include <linux/fsverity.h>
#include "internal.h"
#include "trace.h"
@ -353,9 +354,26 @@ static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
{
const struct iomap *srcmap = iomap_iter_srcmap(iter);
return srcmap->type != IOMAP_MAPPED ||
(srcmap->flags & IOMAP_F_NEW) ||
pos >= i_size_read(iter->inode);
/*
* If this block has not been written, there's nothing to read
*/
if (srcmap->type != IOMAP_MAPPED)
return true;
/*
* Newly allocated blocks have not been written
*/
if (srcmap->flags & IOMAP_F_NEW)
return true;
/*
* fsverity metadata is stored past i_size, we need to read it instead
* of zeroing
*/
if (srcmap->flags & IOMAP_F_FSVERITY)
return false;
return pos >= i_size_read(iter->inode);
}
/**
@ -544,9 +562,27 @@ static int iomap_read_folio_iter(struct iomap_iter *iter,
if (plen == 0)
return 0;
/* zero post-eof blocks as the page may be mapped */
if (iomap_block_needs_zeroing(iter, pos)) {
/*
* Handling of fsverity "holes". We hit this for two case:
* 1. No need to go further, the hole after fsverity
* descriptor is the end of the fsverity metadata.
*
* 2. This folio contains merkle tree blocks which need to be
* synthesized. If we already have fsverity info (ctx->vi)
* synthesize these blocks.
*/
if ((iomap->flags & IOMAP_F_FSVERITY) &&
iomap->type == IOMAP_HOLE) {
if (ctx->vi)
fsverity_fill_zerohash(folio, poff, plen,
ctx->vi);
iomap_set_range_uptodate(folio, poff, plen);
} else if (iomap_block_needs_zeroing(iter, pos)) {
/* zero post-eof blocks as the page may be mapped */
folio_zero_range(folio, poff, plen);
if (ctx->vi &&
!fsverity_verify_blocks(ctx->vi, folio, plen, poff))
return -EIO;
iomap_set_range_uptodate(folio, poff, plen);
} else {
if (!*bytes_submitted)
@ -597,6 +633,15 @@ void iomap_read_folio(const struct iomap_ops *ops,
trace_iomap_readpage(iter.inode, 1);
/*
* Fetch fsverity_info for both data and fsverity metadata, as iomap
* needs zeroed hash for merkle tree block synthesis
*/
ctx->vi = fsverity_get_info(iter.inode);
if (ctx->vi && iter.pos < i_size_read(iter.inode))
fsverity_readahead(ctx->vi, folio->index,
folio_nr_pages(folio));
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.status = iomap_read_folio_iter(&iter, ctx,
&bytes_submitted);
@ -664,6 +709,15 @@ void iomap_readahead(const struct iomap_ops *ops,
trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
/*
* Fetch fsverity_info for both data and fsverity metadata, as iomap
* needs zeroed hash for merkle tree block synthesis
*/
ctx->vi = fsverity_get_info(iter.inode);
if (ctx->vi && iter.pos < i_size_read(iter.inode))
fsverity_readahead(ctx->vi, readahead_index(rac),
readahead_count(rac));
while (iomap_iter(&iter, ops) > 0)
iter.status = iomap_readahead_iter(&iter, ctx,
&cur_bytes_submitted);
@ -836,6 +890,7 @@ static int __iomap_write_begin(const struct iomap_iter *iter,
return -EIO;
folio_zero_segments(folio, poff, from, to, poff + plen);
} else {
const struct iomap *iomap = iomap_iter_srcmap(iter);
int status;
if (iter->flags & IOMAP_NOWAIT)
@ -853,6 +908,9 @@ static int __iomap_write_begin(const struct iomap_iter *iter,
len, status, GFP_NOFS);
if (status)
return status;
if (iomap->flags & IOMAP_F_ZERO_TAIL)
folio_zero_segment(folio, to, poff + plen);
}
iomap_set_range_uptodate(folio, poff, plen);
} while ((block_start += plen) < block_end);
@ -1058,7 +1116,6 @@ static bool iomap_write_end_inline(const struct iomap_iter *iter,
void *addr;
WARN_ON_ONCE(!folio_test_uptodate(folio));
BUG_ON(!iomap_inline_data_valid(iomap));
if (WARN_ON_ONCE(!iomap->inline_data))
return false;
@ -1167,13 +1224,14 @@ static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i,
* unlock and release the folio.
*/
old_size = iter->inode->i_size;
if (pos + written > old_size) {
if (pos + written > old_size &&
!(iter->iomap.flags & IOMAP_F_FSVERITY)) {
i_size_write(iter->inode, pos + written);
iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
}
__iomap_put_folio(iter, write_ops, written, folio);
if (old_size < pos)
if (old_size < pos && !(iter->iomap.flags & IOMAP_F_FSVERITY))
pagecache_isize_extended(iter->inode, old_size, pos);
cond_resched();
@ -1232,6 +1290,31 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
}
EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
int iomap_fsverity_write(struct file *file, loff_t pos, size_t length,
const void *buf, const struct iomap_ops *ops,
const struct iomap_write_ops *write_ops)
{
int ret;
struct iov_iter iiter;
struct kvec kvec = {
.iov_base = (void *)buf,
.iov_len = length,
};
struct kiocb iocb = {
.ki_filp = file,
.ki_ioprio = get_current_ioprio(),
.ki_pos = pos,
};
iov_iter_kvec(&iiter, WRITE, &kvec, 1, length);
ret = iomap_file_buffered_write(&iocb, &iiter, ops, write_ops, NULL);
if (ret < 0)
return ret;
return ret == length ? 0 : -EIO;
}
EXPORT_SYMBOL_GPL(iomap_fsverity_write);
static void iomap_write_delalloc_ifs_punch(struct inode *inode,
struct folio *folio, loff_t start_byte, loff_t end_byte,
struct iomap *iomap, iomap_punch_t punch)
@ -1543,6 +1626,8 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero,
size_t offset;
bool ret;
balance_dirty_pages_ratelimited(iter->inode->i_mapping);
bytes = min_t(u64, SIZE_MAX, bytes);
status = iomap_write_begin(iter, write_ops, &folio, &offset,
&bytes);
@ -1797,13 +1882,20 @@ static int iomap_writeback_range(struct iomap_writepage_ctx *wpc,
* Check interaction of the folio with the file end.
*
* If the folio is entirely beyond i_size, return false. If it straddles
* i_size, adjust end_pos and zero all data beyond i_size.
* i_size, adjust end_pos and zero all data beyond i_size. Don't skip fsverity
* folios as those are beyond i_size.
*/
static bool iomap_writeback_handle_eof(struct folio *folio, struct inode *inode,
u64 *end_pos)
static bool iomap_writeback_handle_eof(struct folio *folio,
struct iomap_writepage_ctx *wpc, u64 *end_pos)
{
struct inode *inode = wpc->inode;
u64 isize = i_size_read(inode);
if (wpc->iomap.flags & IOMAP_F_FSVERITY) {
WARN_ON_ONCE(folio_pos(folio) < isize);
return true;
}
if (*end_pos > isize) {
size_t poff = offset_in_folio(folio, isize);
pgoff_t end_index = isize >> PAGE_SHIFT;
@ -1869,7 +1961,7 @@ int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio)
trace_iomap_writeback_folio(inode, pos, folio_size(folio));
if (!iomap_writeback_handle_eof(folio, inode, &end_pos))
if (!iomap_writeback_handle_eof(folio, wpc, &end_pos))
return 0;
WARN_ON_ONCE(end_pos <= pos);

5
fs/iomap/direct-io.c
View File

@ -69,7 +69,7 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter,
/* Sync dio can't be polled reliably */
if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) {
bio_set_polled(bio, iocb);
bio->bi_opf |= REQ_POLLED;
WRITE_ONCE(iocb->private, bio);
}
@ -603,9 +603,6 @@ static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio)
if (WARN_ON_ONCE(!inline_data))
return -EIO;
if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
return -EIO;
if (dio->flags & IOMAP_DIO_WRITE) {
loff_t size = iomi->inode->i_size;

1
fs/iomap/ioend.c
View File

@ -28,6 +28,7 @@ struct iomap_ioend *iomap_init_ioend(struct inode *inode,
ioend->io_offset = file_offset;
ioend->io_size = bio->bi_iter.bi_size;
ioend->io_sector = bio->bi_iter.bi_sector;
ioend->io_vi = NULL;
ioend->io_private = NULL;
return ioend;
}

12
fs/iomap/iter.c
View File

@ -6,17 +6,13 @@
#include <linux/iomap.h>
#include "trace.h"
static inline void iomap_iter_reset_iomap(struct iomap_iter *iter)
static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter)
{
if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
folio_batch_release(iter->fbatch);
folio_batch_reinit(iter->fbatch);
iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH;
}
iter->status = 0;
memset(&iter->iomap, 0, sizeof(iter->iomap));
memset(&iter->srcmap, 0, sizeof(iter->srcmap));
}
/* Advance the current iterator position and decrement the remaining length */
@ -102,10 +98,14 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
ret = 0;
else
ret = 1;
iomap_iter_reset_iomap(iter);
iomap_iter_clean_fbatch(iter);
iter->status = 0;
if (ret <= 0)
return ret;
memset(&iter->iomap, 0, sizeof(iter->iomap));
memset(&iter->srcmap, 0, sizeof(iter->srcmap));
begin:
ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
&iter->iomap, &iter->srcmap);

4
fs/iomap/trace.h
View File

@ -118,7 +118,9 @@ DEFINE_RANGE_EVENT(iomap_zero_iter);
{ IOMAP_F_ATOMIC_BIO, "ATOMIC_BIO" }, \
{ IOMAP_F_PRIVATE, "PRIVATE" }, \
{ IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }, \
{ IOMAP_F_STALE, "STALE" }
{ IOMAP_F_STALE, "STALE" }, \
{ IOMAP_F_FSVERITY, "FSVERITY" }, \
{ IOMAP_F_ZERO_TAIL, "ZERO TAIL" }
#define IOMAP_DIO_STRINGS \

3
fs/verity/fsverity_private.h
View File

@ -53,6 +53,9 @@ struct merkle_tree_params {
u64 tree_size; /* Merkle tree size in bytes */
unsigned long tree_pages; /* Merkle tree size in pages */
/* the hash of an all-zeroes block */
u8 zero_digest[FS_VERITY_MAX_DIGEST_SIZE];
/*
* Starting block index for each tree level, ordered from leaf level (0)
* to root level ('num_levels - 1')

4
fs/verity/measure.c
View File

@ -68,8 +68,8 @@ EXPORT_SYMBOL_GPL(fsverity_ioctl_measure);
* @alg: (out) the digest's algorithm, as a FS_VERITY_HASH_ALG_* value
* @halg: (out) the digest's algorithm, as a HASH_ALGO_* value
*
* Retrieves the fsverity digest of the given file. The file must have been
* opened at least once since the inode was last loaded into the inode cache;
* Retrieves the fsverity digest of the given file. The
* fsverity_ensure_verity_info() must be called on the inode beforehand;
* otherwise this function will not recognize when fsverity is enabled.
*
* The file's fsverity digest consists of @raw_digest in combination with either

3
fs/verity/open.c
View File

@ -153,6 +153,9 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params,
goto out_err;
}
fsverity_hash_block(params, page_address(ZERO_PAGE(0)),
params->zero_digest);
params->tree_size = offset << log_blocksize;
params->tree_pages = PAGE_ALIGN(params->tree_size) >> PAGE_SHIFT;
return 0;

22
fs/verity/pagecache.c
View File

@ -2,6 +2,7 @@
/*
* Copyright 2019 Google LLC
*/
#include "fsverity_private.h"
#include <linux/export.h>
#include <linux/fsverity.h>
@ -56,3 +57,24 @@ void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index,
folio_put(folio);
}
EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree);
/**
* fsverity_fill_zerohash() - fill folio with hashes of zero data block
* @folio: folio to fill
* @offset: offset in the folio to start
* @len: length of the range to fill with hashes
* @vi: fsverity info
*/
void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len,
struct fsverity_info *vi)
{
size_t off = offset;
WARN_ON_ONCE(!IS_ALIGNED(offset, vi->tree_params.digest_size));
WARN_ON_ONCE(!IS_ALIGNED(len, vi->tree_params.digest_size));
for (; off < (offset + len); off += vi->tree_params.digest_size)
memcpy_to_folio(folio, off, vi->tree_params.zero_digest,
vi->tree_params.digest_size);
}
EXPORT_SYMBOL_GPL(fsverity_fill_zerohash);

14
include/linux/bio.h
View File

@ -703,20 +703,6 @@ static inline bool bioset_initialized(struct bio_set *bs)
return bs->bio_slab != NULL;
}
/*
* Mark a bio as polled. Note that for async polled IO, the caller must
* expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
* We cannot block waiting for requests on polled IO, as those completions
* must be found by the caller. This is different than IRQ driven IO, where
* it's safe to wait for IO to complete.
*/
static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
{
bio->bi_opf |= REQ_POLLED;
if (kiocb->ki_flags & IOCB_NOWAIT)
bio->bi_opf |= REQ_NOWAIT;
}
static inline void bio_clear_polled(struct bio *bio)
{
bio->bi_opf &= ~REQ_POLLED;

8
include/linux/fsverity.h
View File

@ -201,6 +201,8 @@ bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio,
size_t len, size_t offset);
void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio);
void fsverity_enqueue_verify_work(struct work_struct *work);
void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len,
struct fsverity_info *vi);
#else /* !CONFIG_FS_VERITY */
@ -281,6 +283,12 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work)
WARN_ON_ONCE(1);
}
static inline void fsverity_fill_zerohash(struct folio *folio, size_t offset,
size_t len, struct fsverity_info *vi)
{
WARN_ON_ONCE(1);
}
#endif /* !CONFIG_FS_VERITY */
static inline bool fsverity_verify_folio(struct fsverity_info *vi,

27
include/linux/iomap.h
View File

@ -67,6 +67,9 @@ struct vm_fault;
* bio, i.e. set REQ_ATOMIC.
*
* IOMAP_F_INTEGRITY indicates that the filesystems handles integrity metadata.
*
* IOMAP_F_ZERO_TAIL indicates the remainder of the block after the data
* written should be zeroed.
*/
#define IOMAP_F_NEW (1U << 0)
#define IOMAP_F_DIRTY (1U << 1)
@ -86,6 +89,15 @@ struct vm_fault;
#else
#define IOMAP_F_INTEGRITY 0
#endif /* CONFIG_BLK_DEV_INTEGRITY */
#define IOMAP_F_ZERO_TAIL (1U << 10)
/*
* Indicates reads and writes of fsverity metadata.
*
* Fsverity metadata is stored after the regular file data and thus beyond
* i_size.
*/
#define IOMAP_F_FSVERITY (1U << 11)
/*
* Flag reserved for file system specific usage
@ -142,16 +154,6 @@ static inline void *iomap_inline_data(const struct iomap *iomap, loff_t pos)
return iomap->inline_data + pos - iomap->offset;
}
/*
* Check if the mapping's length is within the valid range for inline data.
* This is used to guard against accessing data beyond the page inline_data
* points at.
*/
static inline bool iomap_inline_data_valid(const struct iomap *iomap)
{
return iomap->length <= PAGE_SIZE - offset_in_page(iomap->inline_data);
}
/*
* When get_folio succeeds, put_folio will always be called to do any
* cleanup work necessary. put_folio is responsible for unlocking and putting
@ -351,6 +353,9 @@ static inline bool iomap_want_unshare_iter(const struct iomap_iter *iter)
ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from,
const struct iomap_ops *ops,
const struct iomap_write_ops *write_ops, void *private);
int iomap_fsverity_write(struct file *file, loff_t pos, size_t length,
const void *buf, const struct iomap_ops *ops,
const struct iomap_write_ops *write_ops);
void iomap_read_folio(const struct iomap_ops *ops,
struct iomap_read_folio_ctx *ctx, void *private);
void iomap_readahead(const struct iomap_ops *ops,
@ -427,6 +432,7 @@ struct iomap_ioend {
loff_t io_offset; /* offset in the file */
sector_t io_sector; /* start sector of ioend */
void *io_private; /* file system private data */
struct fsverity_info *io_vi; /* fsverity info */
struct bio io_bio; /* MUST BE LAST! */
};
@ -501,6 +507,7 @@ struct iomap_read_folio_ctx {
struct readahead_control *rac;
void *read_ctx;
loff_t read_ctx_file_offset;
struct fsverity_info *vi;
};
struct iomap_read_ops {