diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index d55b936e6986..5a107d59ae27 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "internal.h" #include "trace.h" @@ -353,9 +354,26 @@ static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter, { const struct iomap *srcmap = iomap_iter_srcmap(iter); - return srcmap->type != IOMAP_MAPPED || - (srcmap->flags & IOMAP_F_NEW) || - pos >= i_size_read(iter->inode); + /* + * If this block has not been written, there's nothing to read + */ + if (srcmap->type != IOMAP_MAPPED) + return true; + + /* + * Newly allocated blocks have not been written + */ + if (srcmap->flags & IOMAP_F_NEW) + return true; + + /* + * fsverity metadata is stored past i_size, we need to read it instead + * of zeroing + */ + if (srcmap->flags & IOMAP_F_FSVERITY) + return false; + + return pos >= i_size_read(iter->inode); } /** @@ -544,9 +562,27 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, if (plen == 0) return 0; - /* zero post-eof blocks as the page may be mapped */ - if (iomap_block_needs_zeroing(iter, pos)) { + /* + * Handling of fsverity "holes". We hit this for two case: + * 1. No need to go further, the hole after fsverity + * descriptor is the end of the fsverity metadata. + * + * 2. This folio contains merkle tree blocks which need to be + * synthesized. If we already have fsverity info (ctx->vi) + * synthesize these blocks. + */ + if ((iomap->flags & IOMAP_F_FSVERITY) && + iomap->type == IOMAP_HOLE) { + if (ctx->vi) + fsverity_fill_zerohash(folio, poff, plen, + ctx->vi); + iomap_set_range_uptodate(folio, poff, plen); + } else if (iomap_block_needs_zeroing(iter, pos)) { + /* zero post-eof blocks as the page may be mapped */ folio_zero_range(folio, poff, plen); + if (ctx->vi && + !fsverity_verify_blocks(ctx->vi, folio, plen, poff)) + return -EIO; iomap_set_range_uptodate(folio, poff, plen); } else { if (!*bytes_submitted) @@ -597,6 +633,15 @@ void iomap_read_folio(const struct iomap_ops *ops, trace_iomap_readpage(iter.inode, 1); + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, folio->index, + folio_nr_pages(folio)); + while ((ret = iomap_iter(&iter, ops)) > 0) iter.status = iomap_read_folio_iter(&iter, ctx, &bytes_submitted); @@ -664,6 +709,15 @@ void iomap_readahead(const struct iomap_ops *ops, trace_iomap_readahead(rac->mapping->host, readahead_count(rac)); + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, readahead_index(rac), + readahead_count(rac)); + while (iomap_iter(&iter, ops) > 0) iter.status = iomap_readahead_iter(&iter, ctx, &cur_bytes_submitted); @@ -836,6 +890,7 @@ static int __iomap_write_begin(const struct iomap_iter *iter, return -EIO; folio_zero_segments(folio, poff, from, to, poff + plen); } else { + const struct iomap *iomap = iomap_iter_srcmap(iter); int status; if (iter->flags & IOMAP_NOWAIT) @@ -853,6 +908,9 @@ static int __iomap_write_begin(const struct iomap_iter *iter, len, status, GFP_NOFS); if (status) return status; + + if (iomap->flags & IOMAP_F_ZERO_TAIL) + folio_zero_segment(folio, to, poff + plen); } iomap_set_range_uptodate(folio, poff, plen); } while ((block_start += plen) < block_end); @@ -1058,7 +1116,6 @@ static bool iomap_write_end_inline(const struct iomap_iter *iter, void *addr; WARN_ON_ONCE(!folio_test_uptodate(folio)); - BUG_ON(!iomap_inline_data_valid(iomap)); if (WARN_ON_ONCE(!iomap->inline_data)) return false; @@ -1167,13 +1224,14 @@ static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i, * unlock and release the folio. */ old_size = iter->inode->i_size; - if (pos + written > old_size) { + if (pos + written > old_size && + !(iter->iomap.flags & IOMAP_F_FSVERITY)) { i_size_write(iter->inode, pos + written); iter->iomap.flags |= IOMAP_F_SIZE_CHANGED; } __iomap_put_folio(iter, write_ops, written, folio); - if (old_size < pos) + if (old_size < pos && !(iter->iomap.flags & IOMAP_F_FSVERITY)) pagecache_isize_extended(iter->inode, old_size, pos); cond_resched(); @@ -1232,6 +1290,31 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, } EXPORT_SYMBOL_GPL(iomap_file_buffered_write); +int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, + const void *buf, const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops) +{ + int ret; + struct iov_iter iiter; + struct kvec kvec = { + .iov_base = (void *)buf, + .iov_len = length, + }; + struct kiocb iocb = { + .ki_filp = file, + .ki_ioprio = get_current_ioprio(), + .ki_pos = pos, + }; + + iov_iter_kvec(&iiter, WRITE, &kvec, 1, length); + + ret = iomap_file_buffered_write(&iocb, &iiter, ops, write_ops, NULL); + if (ret < 0) + return ret; + return ret == length ? 0 : -EIO; +} +EXPORT_SYMBOL_GPL(iomap_fsverity_write); + static void iomap_write_delalloc_ifs_punch(struct inode *inode, struct folio *folio, loff_t start_byte, loff_t end_byte, struct iomap *iomap, iomap_punch_t punch) @@ -1543,6 +1626,8 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero, size_t offset; bool ret; + balance_dirty_pages_ratelimited(iter->inode->i_mapping); + bytes = min_t(u64, SIZE_MAX, bytes); status = iomap_write_begin(iter, write_ops, &folio, &offset, &bytes); @@ -1797,13 +1882,20 @@ static int iomap_writeback_range(struct iomap_writepage_ctx *wpc, * Check interaction of the folio with the file end. * * If the folio is entirely beyond i_size, return false. If it straddles - * i_size, adjust end_pos and zero all data beyond i_size. + * i_size, adjust end_pos and zero all data beyond i_size. Don't skip fsverity + * folios as those are beyond i_size. */ -static bool iomap_writeback_handle_eof(struct folio *folio, struct inode *inode, - u64 *end_pos) +static bool iomap_writeback_handle_eof(struct folio *folio, + struct iomap_writepage_ctx *wpc, u64 *end_pos) { + struct inode *inode = wpc->inode; u64 isize = i_size_read(inode); + if (wpc->iomap.flags & IOMAP_F_FSVERITY) { + WARN_ON_ONCE(folio_pos(folio) < isize); + return true; + } + if (*end_pos > isize) { size_t poff = offset_in_folio(folio, isize); pgoff_t end_index = isize >> PAGE_SHIFT; @@ -1869,7 +1961,7 @@ int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio) trace_iomap_writeback_folio(inode, pos, folio_size(folio)); - if (!iomap_writeback_handle_eof(folio, inode, &end_pos)) + if (!iomap_writeback_handle_eof(folio, wpc, &end_pos)) return 0; WARN_ON_ONCE(end_pos <= pos); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b36ee619cdcd..b485e3b191da 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -69,7 +69,7 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter, /* Sync dio can't be polled reliably */ if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) { - bio_set_polled(bio, iocb); + bio->bi_opf |= REQ_POLLED; WRITE_ONCE(iocb->private, bio); } @@ -603,9 +603,6 @@ static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio) if (WARN_ON_ONCE(!inline_data)) return -EIO; - if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap))) - return -EIO; - if (dio->flags & IOMAP_DIO_WRITE) { loff_t size = iomi->inode->i_size; diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index acf3cf98b23a..f7c3e0c70fd7 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -28,6 +28,7 @@ struct iomap_ioend *iomap_init_ioend(struct inode *inode, ioend->io_offset = file_offset; ioend->io_size = bio->bi_iter.bi_size; ioend->io_sector = bio->bi_iter.bi_sector; + ioend->io_vi = NULL; ioend->io_private = NULL; return ioend; } diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index c04796f6e57f..e4a29829591a 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,17 +6,13 @@ #include #include "trace.h" -static inline void iomap_iter_reset_iomap(struct iomap_iter *iter) +static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) { if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } - - iter->status = 0; - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); } /* Advance the current iterator position and decrement the remaining length */ @@ -102,10 +98,14 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) ret = 0; else ret = 1; - iomap_iter_reset_iomap(iter); + iomap_iter_clean_fbatch(iter); + iter->status = 0; if (ret <= 0) return ret; + memset(&iter->iomap, 0, sizeof(iter->iomap)); + memset(&iter->srcmap, 0, sizeof(iter->srcmap)); + begin: ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, &iter->iomap, &iter->srcmap); diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h index 097773c6db80..e1ea8392cf47 100644 --- a/fs/iomap/trace.h +++ b/fs/iomap/trace.h @@ -118,7 +118,9 @@ DEFINE_RANGE_EVENT(iomap_zero_iter); { IOMAP_F_ATOMIC_BIO, "ATOMIC_BIO" }, \ { IOMAP_F_PRIVATE, "PRIVATE" }, \ { IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }, \ - { IOMAP_F_STALE, "STALE" } + { IOMAP_F_STALE, "STALE" }, \ + { IOMAP_F_FSVERITY, "FSVERITY" }, \ + { IOMAP_F_ZERO_TAIL, "ZERO TAIL" } #define IOMAP_DIO_STRINGS \ diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 6e6854c19078..881d46f25e08 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -53,6 +53,9 @@ struct merkle_tree_params { u64 tree_size; /* Merkle tree size in bytes */ unsigned long tree_pages; /* Merkle tree size in pages */ + /* the hash of an all-zeroes block */ + u8 zero_digest[FS_VERITY_MAX_DIGEST_SIZE]; + /* * Starting block index for each tree level, ordered from leaf level (0) * to root level ('num_levels - 1') diff --git a/fs/verity/measure.c b/fs/verity/measure.c index 6a35623ebdf0..818083507885 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -68,8 +68,8 @@ EXPORT_SYMBOL_GPL(fsverity_ioctl_measure); * @alg: (out) the digest's algorithm, as a FS_VERITY_HASH_ALG_* value * @halg: (out) the digest's algorithm, as a HASH_ALGO_* value * - * Retrieves the fsverity digest of the given file. The file must have been - * opened at least once since the inode was last loaded into the inode cache; + * Retrieves the fsverity digest of the given file. The + * fsverity_ensure_verity_info() must be called on the inode beforehand; * otherwise this function will not recognize when fsverity is enabled. * * The file's fsverity digest consists of @raw_digest in combination with either diff --git a/fs/verity/open.c b/fs/verity/open.c index dfa0d1afe0fe..d0c56a7faa3b 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -153,6 +153,9 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, goto out_err; } + fsverity_hash_block(params, page_address(ZERO_PAGE(0)), + params->zero_digest); + params->tree_size = offset << log_blocksize; params->tree_pages = PAGE_ALIGN(params->tree_size) >> PAGE_SHIFT; return 0; diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c index 1819314ecaa3..99f5f53eea98 100644 --- a/fs/verity/pagecache.c +++ b/fs/verity/pagecache.c @@ -2,6 +2,7 @@ /* * Copyright 2019 Google LLC */ +#include "fsverity_private.h" #include #include @@ -56,3 +57,24 @@ void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, folio_put(folio); } EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree); + +/** + * fsverity_fill_zerohash() - fill folio with hashes of zero data block + * @folio: folio to fill + * @offset: offset in the folio to start + * @len: length of the range to fill with hashes + * @vi: fsverity info + */ +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi) +{ + size_t off = offset; + + WARN_ON_ONCE(!IS_ALIGNED(offset, vi->tree_params.digest_size)); + WARN_ON_ONCE(!IS_ALIGNED(len, vi->tree_params.digest_size)); + + for (; off < (offset + len); off += vi->tree_params.digest_size) + memcpy_to_folio(folio, off, vi->tree_params.zero_digest, + vi->tree_params.digest_size); +} +EXPORT_SYMBOL_GPL(fsverity_fill_zerohash); diff --git a/include/linux/bio.h b/include/linux/bio.h index dc17780d6c1e..8300d5565e36 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -703,20 +703,6 @@ static inline bool bioset_initialized(struct bio_set *bs) return bs->bio_slab != NULL; } -/* - * Mark a bio as polled. Note that for async polled IO, the caller must - * expect -EWOULDBLOCK if we cannot allocate a request (or other resources). - * We cannot block waiting for requests on polled IO, as those completions - * must be found by the caller. This is different than IRQ driven IO, where - * it's safe to wait for IO to complete. - */ -static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) -{ - bio->bi_opf |= REQ_POLLED; - if (kiocb->ki_flags & IOCB_NOWAIT) - bio->bi_opf |= REQ_NOWAIT; -} - static inline void bio_clear_polled(struct bio *bio) { bio->bi_opf &= ~REQ_POLLED; diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index a8f9aa75b792..6c467ded9751 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -201,6 +201,8 @@ bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, size_t len, size_t offset); void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio); void fsverity_enqueue_verify_work(struct work_struct *work); +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi); #else /* !CONFIG_FS_VERITY */ @@ -281,6 +283,12 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work) WARN_ON_ONCE(1); } +static inline void fsverity_fill_zerohash(struct folio *folio, size_t offset, + size_t len, struct fsverity_info *vi) +{ + WARN_ON_ONCE(1); +} + #endif /* !CONFIG_FS_VERITY */ static inline bool fsverity_verify_folio(struct fsverity_info *vi, diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 2c5685adf3a9..3582ed1fe236 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -67,6 +67,9 @@ struct vm_fault; * bio, i.e. set REQ_ATOMIC. * * IOMAP_F_INTEGRITY indicates that the filesystems handles integrity metadata. + * + * IOMAP_F_ZERO_TAIL indicates the remainder of the block after the data + * written should be zeroed. */ #define IOMAP_F_NEW (1U << 0) #define IOMAP_F_DIRTY (1U << 1) @@ -86,6 +89,15 @@ struct vm_fault; #else #define IOMAP_F_INTEGRITY 0 #endif /* CONFIG_BLK_DEV_INTEGRITY */ +#define IOMAP_F_ZERO_TAIL (1U << 10) + +/* + * Indicates reads and writes of fsverity metadata. + * + * Fsverity metadata is stored after the regular file data and thus beyond + * i_size. + */ +#define IOMAP_F_FSVERITY (1U << 11) /* * Flag reserved for file system specific usage @@ -142,16 +154,6 @@ static inline void *iomap_inline_data(const struct iomap *iomap, loff_t pos) return iomap->inline_data + pos - iomap->offset; } -/* - * Check if the mapping's length is within the valid range for inline data. - * This is used to guard against accessing data beyond the page inline_data - * points at. - */ -static inline bool iomap_inline_data_valid(const struct iomap *iomap) -{ - return iomap->length <= PAGE_SIZE - offset_in_page(iomap->inline_data); -} - /* * When get_folio succeeds, put_folio will always be called to do any * cleanup work necessary. put_folio is responsible for unlocking and putting @@ -351,6 +353,9 @@ static inline bool iomap_want_unshare_iter(const struct iomap_iter *iter) ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops, void *private); +int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, + const void *buf, const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops); void iomap_read_folio(const struct iomap_ops *ops, struct iomap_read_folio_ctx *ctx, void *private); void iomap_readahead(const struct iomap_ops *ops, @@ -427,6 +432,7 @@ struct iomap_ioend { loff_t io_offset; /* offset in the file */ sector_t io_sector; /* start sector of ioend */ void *io_private; /* file system private data */ + struct fsverity_info *io_vi; /* fsverity info */ struct bio io_bio; /* MUST BE LAST! */ }; @@ -501,6 +507,7 @@ struct iomap_read_folio_ctx { struct readahead_control *rac; void *read_ctx; loff_t read_ctx_file_offset; + struct fsverity_info *vi; }; struct iomap_read_ops {