mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Add support for buffered iomap support in ext4. This is not enabled
by default, but requires the buffered_iomap mount option, as this mode does not support online defragmentation, inline data, fsverify, fscrypt, non-extent inodes, and data=journal mode. If buffered_iomap is enabled, ext4 will fall back to the original I/O code path automatically if necessary. Large writes show significant performance improvements when buffered iomap is enabled. There are also miscellaneous bug fixes and improvements, including for fast commit and avoiding deadlocks and out-of-bound reads when handling corner cases involving corrupted file systems. -----BEGIN PGP SIGNATURE----- iQEzBAABCAAdFiEEK2m5VNv+CHkogTfJ8vlZVpUNgaMFAmraDnsACgkQ8vlZVpUN gaND7Af/Q9kw8G2+5DS1Emi44hWXTsVFnRc9qGIZ3zChYD+oX9IZt+ipSxomFzal DiMTh0mHlwuixz5NdT6QgGUi1TQlOKrtHKV22orhLZBUsLcm0hwmXg8Xxk0pBhTH N8er1fDx2QKEt0p1u5E1g/YT4Mub7Iug5XD0Yzh6qTtboos/WLYt2rGryp49miLA q+4FWFQHYzyqqurIIx7dWdLRvzql+ffI4rEgtTm2yZP4ru8UGQZyK3SL7p/MpIu8 jop2ubj6ESRHis96F5v4FfKyulnunCrSu3V7GmWlC9KG+iu/J/EdtQEioqHPSA1D uimwsmD1JG44iPhYrW8gIN4r0DohtQ== =PNwN -----END PGP SIGNATURE----- Merge tag 'ext4_for_linus-7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4 Pull ext4 updates from Ted Ts'o: "Add support for buffered iomap support in ext4. This is not enabled by default, but requires the buffered_iomap mount option, as this mode does not support online defragmentation, inline data, fsverify, fscrypt, non-extent inodes, and data=journal mode. If buffered_iomap is enabled, ext4 will fall back to the original I/O code path automatically if necessary. Large writes show significant performance improvements when buffered iomap is enabled. There are also miscellaneous bug fixes and improvements, including for fast commit and avoiding deadlocks and out-of-bound reads when handling corner cases involving corrupted file systems" * tag 'ext4_for_linus-7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4: (41 commits) ext4: introduce a mount option for iomap buffered I/O path ext4: partially enable iomap for the buffered I/O path of regular files ext4: add tracepoints for EOF block zeroing and disksize-grow I/O ext4: add tracepoints for DISKSIZE_GROW_PENDING set, clear, and wait ext4: set DISKSIZE_GROW_PENDING after zeroing unaligned EOF block ext4: clear DISKSIZE_GROW_PENDING on truncate or error ext4: submit and wait for disksize-grow I/O in fallocate paths ext4: defer i_disksize update while DISKSIZE_GROW_PENDING is set ext4: advance i_disksize to i_size upon disksize-grow I/O completion ext4: submit and wait for pending disksize-grow I/O on writeback ext4: add EXT4_STATE_DISKSIZE_GROW_PENDING state bit and helpers ext4: disable online defrag when inode using iomap buffered I/O path ext4: add block mapping tracepoints for iomap buffered I/O path ext4: drain writeback before removing extents on the iomap path ext4: implement partial block zero range path using iomap ext4: implement mmap path using iomap ext4: implement writeback path using iomap ext4: rework handle credit accounting for unwritten extent conversion ext4: implement buffered write path using iomap ext4: do not use data=ordered mode for inodes using buffered iomap path ...
This commit is contained in:
Linus Torvalds
commit
ac5feaad48
|
|
@ -329,7 +329,7 @@ struct ext4_group_info *ext4_get_group_info(struct super_block *sb,
|
|||
struct ext4_group_info **grp_info;
|
||||
long indexv, indexh;
|
||||
|
||||
if (unlikely(group >= EXT4_SB(sb)->s_groups_count))
|
||||
if (unlikely(group >= ext4_get_groups_count(sb)))
|
||||
return NULL;
|
||||
if (unlikely(!EXT4_SB(sb)->s_group_info))
|
||||
return NULL;
|
||||
|
|
|
|||
|
|
@ -1208,8 +1208,10 @@ struct ext4_inode_info {
|
|||
/* Lock protecting lists below */
|
||||
spinlock_t i_completed_io_lock;
|
||||
/*
|
||||
* Completed IOs that need unwritten extents handling and have
|
||||
* transaction reserved
|
||||
* Completed IOs that need unwritten extents handling and have a
|
||||
* transaction reserved for the buffer_head writeback path, and
|
||||
* also used by the iomap writeback path to queue ioends needing
|
||||
* unwritten extents conversion, i_disksize update, etc.
|
||||
*/
|
||||
struct list_head i_rsv_conversion_list;
|
||||
struct work_struct i_rsv_conversion_work;
|
||||
|
|
@ -1319,6 +1321,7 @@ struct ext4_inode_info {
|
|||
* scanning in mballoc
|
||||
*/
|
||||
#define EXT4_MOUNT2_ABORT 0x00000100 /* Abort filesystem */
|
||||
#define EXT4_MOUNT2_BUFFERED_IOMAP 0x00000200 /* Use iomap for buffered I/O */
|
||||
|
||||
#define clear_opt(sb, opt) EXT4_SB(sb)->s_mount_opt &= \
|
||||
~EXT4_MOUNT_##opt
|
||||
|
|
@ -2051,6 +2054,10 @@ enum {
|
|||
EXT4_STATE_FC_FLUSHING_DATA, /* Fast commit flushing data */
|
||||
EXT4_STATE_ORPHAN_FILE, /* Inode orphaned in orphan file */
|
||||
EXT4_STATE_FC_REQUEUE, /* Inode modified during fast commit */
|
||||
EXT4_STATE_BUFFERED_IOMAP, /* Inode use iomap for buffered IO */
|
||||
EXT4_STATE_DISKSIZE_GROW_PENDING,
|
||||
/* Has zeroed EOF block straddles
|
||||
* i_disksize awaiting writeback */
|
||||
};
|
||||
|
||||
#define EXT4_INODE_BIT_FNS(name, field, offset) \
|
||||
|
|
@ -2150,6 +2157,12 @@ static inline struct mapping_metadata_bhs *ext4_i_metadata_bhs(
|
|||
return READ_ONCE(EXT4_I(inode)->i_metadata_bhs);
|
||||
}
|
||||
|
||||
/* Whether the inode pass through the iomap infrastructure for buffered I/O */
|
||||
static inline bool ext4_inode_buffered_iomap(struct inode *inode)
|
||||
{
|
||||
return ext4_test_inode_state(inode, EXT4_STATE_BUFFERED_IOMAP);
|
||||
}
|
||||
|
||||
/*
|
||||
* Codes for operating systems
|
||||
*/
|
||||
|
|
@ -3160,6 +3173,8 @@ int ext4_walk_page_buffers(handle_t *handle,
|
|||
int do_journal_get_write_access(handle_t *handle, struct inode *inode,
|
||||
struct buffer_head *bh);
|
||||
void ext4_set_inode_mapping_order(struct inode *inode);
|
||||
void ext4_enable_buffered_iomap(struct inode *inode);
|
||||
int ext4_nonda_switch(struct super_block *sb);
|
||||
#define FALL_BACK_TO_NONDELALLOC 1
|
||||
#define EXT4_WRITE_DATA_INLINE 2
|
||||
|
||||
|
|
@ -3211,6 +3226,12 @@ extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks);
|
|||
extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks);
|
||||
extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
|
||||
int pextents, int alloc_extents);
|
||||
void ext4_iomap_clear_disksize_pending(struct inode *inode);
|
||||
void ext4_iomap_wait_disksize_pending(struct inode *inode);
|
||||
unsigned int ext4_iomap_get_disksize_pending_range(struct inode *inode,
|
||||
loff_t *start);
|
||||
extern int ext4_iomap_sync_zeroed_eof(struct inode *inode,
|
||||
loff_t offset, loff_t end);
|
||||
extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end);
|
||||
|
||||
#define EXT4_PARTIAL_ZERO_START 0x1
|
||||
|
|
@ -3544,16 +3565,12 @@ static inline void ext4_isize_set(struct ext4_inode *raw_inode, loff_t i_size)
|
|||
}
|
||||
|
||||
/*
|
||||
* Reading s_groups_count requires using smp_rmb() afterwards. See
|
||||
* the locking protocol documented in the comments of ext4_group_add()
|
||||
* in resize.c
|
||||
* Reading s_groups_count uses acquire semantics.
|
||||
*/
|
||||
static inline ext4_group_t ext4_get_groups_count(struct super_block *sb)
|
||||
{
|
||||
ext4_group_t ngroups = EXT4_SB(sb)->s_groups_count;
|
||||
|
||||
smp_rmb();
|
||||
return ngroups;
|
||||
/* Pairs with smp_store_release() in ext4_update_super() */
|
||||
return smp_load_acquire(&EXT4_SB(sb)->s_groups_count);
|
||||
}
|
||||
|
||||
static inline ext4_group_t ext4_flex_group(struct ext4_sb_info *sbi,
|
||||
|
|
@ -3590,30 +3607,67 @@ do { \
|
|||
#define EXT4_FREECLUSTERS_WATERMARK 0
|
||||
#endif
|
||||
|
||||
/* Update i_disksize. Requires i_rwsem to avoid races with truncate */
|
||||
/*
|
||||
* Update i_disksize. Requires i_rwsem to avoid races with truncate.
|
||||
*
|
||||
* In the iomap buffered I/O path, the EXT4_STATE_DISKSIZE_GROW_PENDING
|
||||
* inode state bit indicates that the zeroed EOF partial block which
|
||||
* straddles i_disksize is still waiting writeback. In that case,
|
||||
* i_disksize will be updated after the pending zeroed data has been
|
||||
* written out.
|
||||
*/
|
||||
static inline void ext4_update_i_disksize(struct inode *inode, loff_t newsize)
|
||||
{
|
||||
WARN_ON_ONCE(S_ISREG(inode->i_mode) &&
|
||||
!inode_is_locked(inode));
|
||||
down_write(&EXT4_I(inode)->i_data_sem);
|
||||
if (newsize > EXT4_I(inode)->i_disksize)
|
||||
if (newsize > EXT4_I(inode)->i_disksize &&
|
||||
!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
|
||||
WRITE_ONCE(EXT4_I(inode)->i_disksize, newsize);
|
||||
up_write(&EXT4_I(inode)->i_data_sem);
|
||||
}
|
||||
|
||||
/* Update i_size, i_disksize. Requires i_rwsem to avoid races with truncate */
|
||||
static inline void __ext4_set_i_disksize(struct inode *inode, loff_t newsize)
|
||||
{
|
||||
WARN_ON_ONCE(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
|
||||
|
||||
if (newsize < EXT4_I(inode)->i_disksize ||
|
||||
!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
|
||||
WRITE_ONCE(EXT4_I(inode)->i_disksize, newsize);
|
||||
}
|
||||
|
||||
/*
|
||||
* Update i_size and i_disksize to @newsize. Requires i_rwsem to avoid
|
||||
* races with truncate.
|
||||
*
|
||||
* In the iomap buffered I/O path, i_disksize is updated only if no zeroed
|
||||
* pending block straddles i_disksize (EXT4_STATE_DISKSIZE_GROW_PENDING
|
||||
* clear), otherwise the ioend worker for the pending block will advance
|
||||
* i_disksize once the pending block is written back. Both updates happen
|
||||
* under i_data_sem so that the writeback ioend worker can always see the
|
||||
* latest i_size under the same semaphore.
|
||||
*
|
||||
* Returns 0 if nothing changed, 1 if i_size was raised, 2 if i_disksize
|
||||
* was raised, or 3 if both were.
|
||||
*/
|
||||
static inline int ext4_update_inode_size(struct inode *inode, loff_t newsize)
|
||||
{
|
||||
int changed = 0;
|
||||
|
||||
if (newsize <= inode->i_size && newsize <= EXT4_I(inode)->i_disksize)
|
||||
return 0;
|
||||
|
||||
down_write(&EXT4_I(inode)->i_data_sem);
|
||||
if (newsize > inode->i_size) {
|
||||
i_size_write(inode, newsize);
|
||||
changed = 1;
|
||||
}
|
||||
if (newsize > EXT4_I(inode)->i_disksize) {
|
||||
ext4_update_i_disksize(inode, newsize);
|
||||
if (newsize > EXT4_I(inode)->i_disksize &&
|
||||
!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING)) {
|
||||
WRITE_ONCE(EXT4_I(inode)->i_disksize, newsize);
|
||||
changed |= 2;
|
||||
}
|
||||
up_write(&EXT4_I(inode)->i_data_sem);
|
||||
return changed;
|
||||
}
|
||||
|
||||
|
|
@ -3907,6 +3961,9 @@ extern void ext4_ext_tree_init(handle_t *handle, struct inode *inode);
|
|||
extern int ext4_ext_index_trans_blocks(struct inode *inode, int extents);
|
||||
extern int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
|
||||
struct ext4_map_blocks *map, int flags);
|
||||
ext4_lblk_t ext4_determine_insert_hole(struct inode *inode, ext4_lblk_t lblk,
|
||||
ext4_lblk_t hole_start,
|
||||
ext4_lblk_t hole_len);
|
||||
extern int ext4_ext_truncate(handle_t *, struct inode *);
|
||||
extern int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
|
||||
ext4_lblk_t end);
|
||||
|
|
@ -3971,6 +4028,12 @@ extern int ext4_move_extents(struct file *o_filp, struct file *d_filp,
|
|||
__u64 len, __u64 *moved_len);
|
||||
|
||||
/* page-io.c */
|
||||
/*
|
||||
* The I/O range covers the zeroed EOF block that straddles i_disksize
|
||||
* and will advance it upon completion.
|
||||
*/
|
||||
#define EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO 1UL
|
||||
|
||||
extern int __init ext4_init_pageio(void);
|
||||
extern void ext4_exit_pageio(void);
|
||||
extern ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags);
|
||||
|
|
@ -3985,6 +4048,8 @@ void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *page,
|
|||
size_t len);
|
||||
extern struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end);
|
||||
extern struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end);
|
||||
extern void ext4_iomap_end_io(struct work_struct *work);
|
||||
extern void ext4_iomap_end_bio(struct bio *bio);
|
||||
|
||||
/* mmp.c */
|
||||
extern int ext4_multi_mount_protect(struct super_block *, ext4_fsblk_t);
|
||||
|
|
@ -4042,6 +4107,9 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end)
|
|||
|
||||
extern const struct iomap_ops ext4_iomap_ops;
|
||||
extern const struct iomap_ops ext4_iomap_report_ops;
|
||||
extern const struct iomap_ops ext4_iomap_buffered_write_ops;
|
||||
extern const struct iomap_ops ext4_iomap_buffered_da_write_ops;
|
||||
extern const struct iomap_write_ops ext4_iomap_write_ops;
|
||||
|
||||
int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
|
||||
unsigned flags, struct iomap *iomap, struct iomap *srcmap);
|
||||
|
|
|
|||
|
|
@ -17,8 +17,12 @@ int ext4_inode_journal_mode(struct inode *inode)
|
|||
test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA ||
|
||||
(ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) &&
|
||||
!test_opt(inode->i_sb, DELALLOC))) {
|
||||
/* We do not support data journalling for encrypted data */
|
||||
if (S_ISREG(inode->i_mode) && IS_ENCRYPTED(inode))
|
||||
/*
|
||||
* We do not support data journalling for encrypted data
|
||||
* and buffered IOMAP path.
|
||||
*/
|
||||
if (S_ISREG(inode->i_mode) &&
|
||||
(IS_ENCRYPTED(inode) || ext4_inode_buffered_iomap(inode)))
|
||||
return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */
|
||||
return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */
|
||||
}
|
||||
|
|
|
|||
|
|
@ -379,7 +379,12 @@ static inline int ext4_should_journal_data(struct inode *inode)
|
|||
|
||||
static inline int ext4_should_order_data(struct inode *inode)
|
||||
{
|
||||
return ext4_inode_journal_mode(inode) & EXT4_INODE_ORDERED_DATA_MODE;
|
||||
/*
|
||||
* inodes using the iomap buffered I/O path do not use the
|
||||
* data=ordered mode.
|
||||
*/
|
||||
return !ext4_inode_buffered_iomap(inode) &&
|
||||
(ext4_inode_journal_mode(inode) & EXT4_INODE_ORDERED_DATA_MODE);
|
||||
}
|
||||
|
||||
static inline int ext4_should_writeback_data(struct inode *inode)
|
||||
|
|
|
|||
|
|
@ -4190,21 +4190,19 @@ static int get_implied_cluster_alloc(struct super_block *sb,
|
|||
}
|
||||
|
||||
/*
|
||||
* Determine hole length around the given logical block, first try to
|
||||
* locate and expand the hole from the given @path, and then adjust it
|
||||
* if it's partially or completely converted to delayed extents, insert
|
||||
* it into the extent cache tree if it's indeed a hole, finally return
|
||||
* the length of the determined extent.
|
||||
* Adjust the hole [@hole_start, @hole_start + @hole_len) the caller found
|
||||
* around the queried block @lblk if it's partially or completely converted
|
||||
* to delayed extents, insert it into the extent cache tree if it's indeed a
|
||||
* hole, finally return the length of the determined extent starting at
|
||||
* @lblk. @hole_start must not be behind @lblk.
|
||||
*/
|
||||
static ext4_lblk_t ext4_ext_determine_insert_hole(struct inode *inode,
|
||||
struct ext4_ext_path *path,
|
||||
ext4_lblk_t lblk)
|
||||
ext4_lblk_t ext4_determine_insert_hole(struct inode *inode, ext4_lblk_t lblk,
|
||||
ext4_lblk_t hole_start,
|
||||
ext4_lblk_t hole_len)
|
||||
{
|
||||
ext4_lblk_t hole_start, len;
|
||||
ext4_lblk_t len = hole_len;
|
||||
struct extent_status es;
|
||||
|
||||
hole_start = lblk;
|
||||
len = ext4_ext_find_hole(inode, path, &hole_start);
|
||||
again:
|
||||
ext4_es_find_extent_range(inode, &ext4_es_is_delayed, hole_start,
|
||||
hole_start + len - 1, &es);
|
||||
|
|
@ -4371,9 +4369,11 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
|
|||
* we couldn't try to create block if flags doesn't contain EXT4_GET_BLOCKS_CREATE
|
||||
*/
|
||||
if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
|
||||
ext4_lblk_t len;
|
||||
ext4_lblk_t hole_start = map->m_lblk, len;
|
||||
|
||||
len = ext4_ext_determine_insert_hole(inode, path, map->m_lblk);
|
||||
len = ext4_ext_find_hole(inode, path, &hole_start);
|
||||
len = ext4_determine_insert_hole(inode, map->m_lblk,
|
||||
hole_start, len);
|
||||
|
||||
map->m_pblk = 0;
|
||||
map->m_len = min_t(unsigned int, map->m_len, len);
|
||||
|
|
@ -4561,7 +4561,7 @@ int ext4_ext_truncate(handle_t *handle, struct inode *inode)
|
|||
*/
|
||||
|
||||
/* we have to know where to truncate from in crash case */
|
||||
EXT4_I(inode)->i_disksize = inode->i_size;
|
||||
__ext4_set_i_disksize(inode, inode->i_size);
|
||||
err = ext4_mark_inode_dirty(handle, inode);
|
||||
if (err)
|
||||
return err;
|
||||
|
|
@ -4857,6 +4857,17 @@ static long ext4_zero_range(struct file *file, loff_t offset,
|
|||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* In SYNC mode, sync the pending zeroed EOF block to ensure the
|
||||
* i_disksize update is persisted.
|
||||
*/
|
||||
if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && new_size) {
|
||||
ret = ext4_iomap_sync_zeroed_eof(inode, 0, LLONG_MAX);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Finish zeroing out if it doesn't contain partial block */
|
||||
if (IS_ALIGNED(offset | end, blocksize))
|
||||
return ret;
|
||||
|
|
@ -4928,10 +4939,20 @@ static long ext4_do_fallocate(struct file *file, loff_t offset,
|
|||
if (ret)
|
||||
goto out;
|
||||
|
||||
if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) &&
|
||||
EXT4_SB(inode->i_sb)->s_journal) {
|
||||
ret = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
|
||||
EXT4_I(inode)->i_sync_tid);
|
||||
if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) {
|
||||
/*
|
||||
* Sync the pending zeroed EOF block to ensure the
|
||||
* i_disksize update is persisted.
|
||||
*/
|
||||
if (new_size) {
|
||||
ret = ext4_iomap_sync_zeroed_eof(inode, 0, LLONG_MAX);
|
||||
if (ret)
|
||||
goto out;
|
||||
}
|
||||
if (EXT4_SB(inode->i_sb)->s_journal) {
|
||||
ret = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
|
||||
EXT4_I(inode)->i_sync_tid);
|
||||
}
|
||||
}
|
||||
out:
|
||||
trace_ext4_fallocate_exit(inode, offset,
|
||||
|
|
@ -5137,27 +5158,36 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
|
|||
int ret = 0, ret2 = 0, ret3 = 0;
|
||||
struct ext4_map_blocks map;
|
||||
unsigned int blkbits = inode->i_blkbits;
|
||||
unsigned int credits = 0;
|
||||
unsigned int credits;
|
||||
bool internal_handle = !handle;
|
||||
|
||||
map.m_lblk = offset >> blkbits;
|
||||
map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
|
||||
|
||||
if (!handle) {
|
||||
/*
|
||||
* credits to insert 1 extent into extent tree
|
||||
*/
|
||||
credits = ext4_chunk_trans_blocks(inode, max_blocks);
|
||||
/* Credits to convert one extent in this range to written state. */
|
||||
credits = ext4_meta_trans_blocks(inode, max_blocks, 1, 0);
|
||||
|
||||
if (internal_handle) {
|
||||
handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits);
|
||||
if (IS_ERR(handle)) {
|
||||
ret = PTR_ERR(handle);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
while (max_blocks) {
|
||||
if (credits) {
|
||||
handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
|
||||
credits);
|
||||
if (IS_ERR(handle)) {
|
||||
ret = PTR_ERR(handle);
|
||||
break;
|
||||
}
|
||||
}
|
||||
/*
|
||||
* The caller cannot know how many extents the range covers,
|
||||
* so make sure the transaction can take one more extent,
|
||||
* extending or restarting it once it is full. The reserved
|
||||
* handle of the buffer_head writeback path is safe as it has
|
||||
* enough credits for the conversion, so it is not restarted
|
||||
* here.
|
||||
*/
|
||||
ret = ext4_journal_ensure_credits(handle, credits, 0);
|
||||
if (ret < 0)
|
||||
break;
|
||||
|
||||
/*
|
||||
* Do not cache any unrelated extents, as it does not hold the
|
||||
* i_rwsem or invalidate_lock, which could corrupt the extent
|
||||
|
|
@ -5177,11 +5207,6 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
|
|||
}
|
||||
|
||||
ret2 = ext4_mark_inode_dirty(handle, inode);
|
||||
if (credits) {
|
||||
ret3 = ext4_journal_stop(handle);
|
||||
if (unlikely(ret3))
|
||||
ret2 = ret3;
|
||||
}
|
||||
ret = ret < 0 ? ret : ret2;
|
||||
if (ret)
|
||||
break;
|
||||
|
|
@ -5189,6 +5214,13 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
|
|||
map.m_lblk += map.m_len;
|
||||
map.m_len = (max_blocks -= map.m_len);
|
||||
}
|
||||
|
||||
if (internal_handle) {
|
||||
ret3 = ext4_journal_stop(handle);
|
||||
if (!ret)
|
||||
ret = ret3;
|
||||
}
|
||||
out:
|
||||
/* Converted some or all blocks successfully? */
|
||||
if (converted)
|
||||
*converted = conv_blocks;
|
||||
|
|
@ -5203,8 +5235,8 @@ int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
|
|||
|
||||
/*
|
||||
* This is somewhat ugly but the idea is clear: When transaction is
|
||||
* reserved, everything goes into it. Otherwise we rather start several
|
||||
* smaller transactions for conversion of each extent separately.
|
||||
* reserved, everything goes into it. Otherwise the conversion runs
|
||||
* its own transaction, extending or restarting it as it goes.
|
||||
*/
|
||||
if (handle) {
|
||||
handle = ext4_journal_start_reserved(handle,
|
||||
|
|
@ -5662,6 +5694,14 @@ static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len)
|
|||
if (end >= inode->i_size)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* Persist the pending zeroed EOF block to ensure i_disksize
|
||||
* can be safely updated thereafter.
|
||||
*/
|
||||
ret = ext4_iomap_sync_zeroed_eof(inode, 0, LLONG_MAX);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/*
|
||||
* Write tail of the last page before removed range and data that
|
||||
* will be shifted since they will get removed from the page cache
|
||||
|
|
@ -5711,7 +5751,7 @@ static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len)
|
|||
|
||||
new_size = inode->i_size - len;
|
||||
i_size_write(inode, new_size);
|
||||
EXT4_I(inode)->i_disksize = new_size;
|
||||
__ext4_set_i_disksize(inode, new_size);
|
||||
|
||||
up_write(&EXT4_I(inode)->i_data_sem);
|
||||
ret = ext4_mark_inode_dirty(handle, inode);
|
||||
|
|
@ -5764,6 +5804,14 @@ static int ext4_insert_range(struct file *file, loff_t offset, loff_t len)
|
|||
if (len > inode->i_sb->s_maxbytes - inode->i_size)
|
||||
return -EFBIG;
|
||||
|
||||
/*
|
||||
* Persist the pending zeroed EOF block to ensure i_disksize
|
||||
* can be safely updated thereafter.
|
||||
*/
|
||||
ret = ext4_iomap_sync_zeroed_eof(inode, 0, LLONG_MAX);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/*
|
||||
* Write out all dirty pages. Need to round down to align start offset
|
||||
* to page size boundary for page size > block size.
|
||||
|
|
@ -5783,8 +5831,7 @@ static int ext4_insert_range(struct file *file, loff_t offset, loff_t len)
|
|||
ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle);
|
||||
|
||||
/* Expand file to avoid data loss if there is error while shifting */
|
||||
inode->i_size += len;
|
||||
EXT4_I(inode)->i_disksize += len;
|
||||
ext4_update_inode_size(inode, inode->i_size + len);
|
||||
ret = ext4_mark_inode_dirty(handle, inode);
|
||||
if (ret)
|
||||
goto out_handle;
|
||||
|
|
|
|||
|
|
@ -330,6 +330,21 @@ static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)
|
|||
return count;
|
||||
}
|
||||
|
||||
static ssize_t ext4_iomap_buffered_write(struct kiocb *iocb,
|
||||
struct iov_iter *from)
|
||||
{
|
||||
struct inode *inode = file_inode(iocb->ki_filp);
|
||||
const struct iomap_ops *iomap_ops;
|
||||
|
||||
if (test_opt(inode->i_sb, DELALLOC) && !ext4_nonda_switch(inode->i_sb))
|
||||
iomap_ops = &ext4_iomap_buffered_da_write_ops;
|
||||
else
|
||||
iomap_ops = &ext4_iomap_buffered_write_ops;
|
||||
|
||||
return iomap_file_buffered_write(iocb, from, iomap_ops,
|
||||
&ext4_iomap_write_ops, NULL);
|
||||
}
|
||||
|
||||
static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
|
||||
struct iov_iter *from)
|
||||
{
|
||||
|
|
@ -351,7 +366,10 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
|
|||
if (ret <= 0)
|
||||
goto out;
|
||||
|
||||
ret = generic_perform_write(iocb, from);
|
||||
if (ext4_inode_buffered_iomap(inode))
|
||||
ret = ext4_iomap_buffered_write(iocb, from);
|
||||
else
|
||||
ret = generic_perform_write(iocb, from);
|
||||
|
||||
out:
|
||||
inode_unlock(inode);
|
||||
|
|
|
|||
|
|
@ -1346,6 +1346,7 @@ struct inode *__ext4_new_inode(const struct mnt_idmap *idmap,
|
|||
}
|
||||
}
|
||||
|
||||
ext4_enable_buffered_iomap(inode);
|
||||
ext4_set_inode_mapping_order(inode);
|
||||
|
||||
ext4_update_inode_fsync_trans(handle, inode, 1);
|
||||
|
|
|
|||
|
|
@ -586,6 +586,23 @@ int ext4_ind_map_blocks(handle_t *handle, struct inode *inode,
|
|||
for (i = partial - chain + 1; i < depth; i++)
|
||||
count = count * epb + (epb - offsets[i] - 1);
|
||||
count++;
|
||||
|
||||
/*
|
||||
* The count knows nothing about delayed allocation, so let
|
||||
* the common helper reconcile it with the extent status
|
||||
* tree. Only when the lookup really did find a hole:
|
||||
* ext4_get_branch() also returns a non-NULL partial when it
|
||||
* failed, and caching a hole for a range it could not read
|
||||
* would answer every later read from that cache instead of
|
||||
* retrying. Clamp the count first, a subtree can be bigger
|
||||
* than the logical block space when the block size is large.
|
||||
*/
|
||||
if (!err) {
|
||||
count = umin(count, EXT_MAX_BLOCKS - map->m_lblk);
|
||||
count = ext4_determine_insert_hole(inode, map->m_lblk,
|
||||
map->m_lblk, count);
|
||||
}
|
||||
|
||||
/* Fill in size of a hole we found */
|
||||
map->m_pblk = 0;
|
||||
map->m_len = umin(map->m_len, count);
|
||||
|
|
|
|||
1202
fs/ext4/inode.c
1202
fs/ext4/inode.c
File diff suppressed because it is too large
Load Diff
|
|
@ -1362,7 +1362,8 @@ static void ext4_sb_setparams(struct ext4_sb_info *sbi,
|
|||
es->s_encoding = cpu_to_le16(params->encoding);
|
||||
if (params->set_flags & EXT4_TUNE_FL_ENCODING_FLAGS)
|
||||
es->s_encoding_flags = cpu_to_le16(params->encoding_flags);
|
||||
strscpy_pad(es->s_mount_opts, params->mount_opts);
|
||||
if (params->set_flags & EXT4_TUNE_FL_MOUNT_OPTS)
|
||||
strscpy_pad(es->s_mount_opts, params->mount_opts);
|
||||
if (params->set_flags & EXT4_TUNE_FL_EDIT_FEATURES) {
|
||||
es->s_feature_compat |=
|
||||
cpu_to_le32(params->set_feature_compat_mask);
|
||||
|
|
|
|||
|
|
@ -899,10 +899,8 @@ static ext4_group_t ext4_get_allocation_groups_count(
|
|||
|
||||
/* non-extent files are limited to low blocks/groups */
|
||||
if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))
|
||||
ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups;
|
||||
|
||||
/* Pairs with smp_wmb() in ext4_update_super() */
|
||||
smp_rmb();
|
||||
/* Pairs with smp_store_release() in ext4_update_super() */
|
||||
ngroups = smp_load_acquire(&EXT4_SB(ac->ac_sb)->s_blockfile_groups);
|
||||
|
||||
return ngroups;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -621,6 +621,8 @@ int ext4_ind_migrate(struct inode *inode)
|
|||
|
||||
if (ext4_has_feature_bigalloc(inode->i_sb))
|
||||
return -EOPNOTSUPP;
|
||||
if (ext4_inode_buffered_iomap(inode))
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
/*
|
||||
* In order to get correct extent info, force all delayed allocation
|
||||
|
|
|
|||
|
|
@ -476,6 +476,17 @@ static int mext_check_validity(struct inode *orig_inode,
|
|||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
/*
|
||||
* TODO: support online defrag for inodes that use the buffered
|
||||
* I/O iomap path.
|
||||
*/
|
||||
if (ext4_inode_buffered_iomap(orig_inode) ||
|
||||
ext4_inode_buffered_iomap(donor_inode)) {
|
||||
ext4_msg(sb, KERN_ERR,
|
||||
"Online defrag not supported for inode with iomap buffered IO path");
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
if (donor_inode->i_mode & (S_ISUID|S_ISGID)) {
|
||||
ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %llu, donor %llu]\n",
|
||||
orig_inode->i_ino, donor_inode->i_ino);
|
||||
|
|
|
|||
|
|
@ -3209,19 +3209,20 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry)
|
|||
ext4_fc_track_unlink(handle, dentry);
|
||||
retval = ext4_mark_inode_dirty(handle, dir);
|
||||
|
||||
end_rmdir:
|
||||
brelse(bh);
|
||||
if (handle)
|
||||
ext4_journal_stop(handle);
|
||||
|
||||
/* VFS negative dentries are incompatible with Encoding and
|
||||
* Case-insensitiveness. Eventually we'll want avoid
|
||||
* invalidating the dentries here, alongside with returning the
|
||||
* negative dentries at ext4_lookup(), when it is better
|
||||
* supported by the VFS for the CI case.
|
||||
*/
|
||||
if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
|
||||
if (!retval && IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
|
||||
d_invalidate(dentry);
|
||||
|
||||
end_rmdir:
|
||||
brelse(bh);
|
||||
if (handle)
|
||||
ext4_journal_stop(handle);
|
||||
return retval;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include <linux/bio.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/iomap.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/sched/mm.h>
|
||||
|
|
@ -30,6 +31,8 @@
|
|||
#include "xattr.h"
|
||||
#include "acl.h"
|
||||
|
||||
#include <trace/events/ext4.h>
|
||||
|
||||
static struct kmem_cache *io_end_cachep;
|
||||
static struct kmem_cache *io_end_vec_cachep;
|
||||
|
||||
|
|
@ -547,3 +550,201 @@ void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio,
|
|||
io_submit_add_bh(io, inode, folio, bh);
|
||||
} while ((bh = bh->b_this_page) != head);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the current writeback range starts beyond the zeroed EOF pending
|
||||
* range that straddles i_disksize, wait for the zeroed data from
|
||||
* ext4_block_zero_eof() to be written out first. Otherwise, extending
|
||||
* i_disksize may expose stale data in the old EOF block.
|
||||
*/
|
||||
static void ext4_iomap_wb_disksize_pending_wait(struct inode *inode,
|
||||
loff_t pos, size_t size)
|
||||
{
|
||||
loff_t disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
|
||||
loff_t pstart, plen;
|
||||
|
||||
/*
|
||||
* Overwrite I/Os and I/Os covering the EOF block do not need to
|
||||
* wait: the former do not advance i_disksize past the pending
|
||||
* boundary, and the latter are the pending I/O itself (cleared in
|
||||
* the bio completion path).
|
||||
*/
|
||||
if (pos < round_up(disksize, i_blocksize(inode)))
|
||||
return;
|
||||
|
||||
plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
|
||||
if (!plen || pos < pstart + plen)
|
||||
return;
|
||||
|
||||
trace_ext4_iomap_wb_disksize_pending_wait(inode, pos, size);
|
||||
ext4_iomap_wait_disksize_pending(inode);
|
||||
}
|
||||
|
||||
static int ext4_iomap_wb_update_disksize(handle_t *handle, struct inode *inode,
|
||||
loff_t end, bool is_disksize_grow)
|
||||
{
|
||||
loff_t new_disksize, i_size;
|
||||
struct ext4_inode_info *ei = EXT4_I(inode);
|
||||
int ret;
|
||||
|
||||
/*
|
||||
* Races with truncate are avoided by checking i_size under
|
||||
* i_data_sem.
|
||||
*/
|
||||
down_write(&ei->i_data_sem);
|
||||
i_size = i_size_read(inode);
|
||||
|
||||
/*
|
||||
* EXT4_STATE_DISKSIZE_GROW_PENDING is cleared when the pending
|
||||
* I/O completes. However, another thread may have re-set the bit
|
||||
* between that point and here, meaning i_disksize has already
|
||||
* been advanced and a new EOF zeroing has been initiated. In that
|
||||
* case, do not advance i_disksize to i_size; leave it to the
|
||||
* next pending grow ioend.
|
||||
*/
|
||||
if (is_disksize_grow &&
|
||||
ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
|
||||
is_disksize_grow = false;
|
||||
|
||||
/*
|
||||
* Update i_disksize to i_size when EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO
|
||||
* completes. This is safe because we never directly allocate written
|
||||
* blocks during buffered writes.
|
||||
*
|
||||
* This ensures that i_disksize is correctly advanced during
|
||||
* truncate-up or append fallocate on a block-unaligned file,
|
||||
* preventing it from remaining stale. The tradeoff is that zeroed
|
||||
* data may be exposed after crash recovery if dirty data in this
|
||||
* range is not yet on disk, but stale data will never be exposed.
|
||||
* This is because the extent is only converted to written state
|
||||
* after the data has been persisted.
|
||||
*/
|
||||
new_disksize = is_disksize_grow ? i_size : min(end, i_size);
|
||||
if (new_disksize > ei->i_disksize) {
|
||||
trace_ext4_iomap_wb_update_disksize(inode, end, i_size,
|
||||
ei->i_disksize, new_disksize, is_disksize_grow);
|
||||
WRITE_ONCE(ei->i_disksize, new_disksize);
|
||||
}
|
||||
up_write(&ei->i_data_sem);
|
||||
ret = ext4_mark_inode_dirty(handle, inode);
|
||||
if (ret)
|
||||
EXT4_ERROR_INODE_ERR(inode, -ret, "Failed to mark inode dirty");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void ext4_iomap_finish_ioend(struct iomap_ioend *ioend)
|
||||
{
|
||||
struct inode *inode = ioend->io_inode;
|
||||
struct super_block *sb = inode->i_sb;
|
||||
loff_t pos = ioend->io_offset;
|
||||
size_t size = ioend->io_size;
|
||||
loff_t end = pos + size;
|
||||
unsigned long io_mode = (unsigned long)ioend->io_private;
|
||||
bool is_disksize_grow = (io_mode == EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO);
|
||||
handle_t *handle;
|
||||
int credits;
|
||||
int ret, err;
|
||||
|
||||
ret = blk_status_to_errno(ioend->io_bio.bi_status);
|
||||
if (is_disksize_grow)
|
||||
trace_ext4_iomap_wb_disksize_pending_complete(ioend->io_inode,
|
||||
ioend->io_offset, ioend->io_size, ret);
|
||||
if (unlikely(ret)) {
|
||||
if (test_opt(sb, DATA_ERR_ABORT) && !ext4_emergency_state(sb))
|
||||
jbd2_journal_abort(EXT4_SB(sb)->s_journal, ret);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (!(ioend->io_flags & IOMAP_IOEND_UNWRITTEN) &&
|
||||
end <= READ_ONCE(EXT4_I(inode)->i_disksize) && !is_disksize_grow)
|
||||
goto out;
|
||||
|
||||
/* Wait for disksize-pending zeroed data to be written out. */
|
||||
ext4_iomap_wb_disksize_pending_wait(inode, pos, size);
|
||||
|
||||
/*
|
||||
* We may need to convert one extent, update the i_disksize and
|
||||
* dirty the inode.
|
||||
*/
|
||||
credits = ext4_chunk_trans_blocks(inode,
|
||||
EXT4_MAX_BLOCKS(size, pos, inode->i_blkbits));
|
||||
handle = ext4_journal_start(inode, EXT4_HT_EXT_CONVERT, credits);
|
||||
if (IS_ERR(handle)) {
|
||||
ret = PTR_ERR(handle);
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
/* Update on-disk size after I/O is completed. */
|
||||
if (end > READ_ONCE(EXT4_I(inode)->i_disksize) || is_disksize_grow) {
|
||||
ret = ext4_iomap_wb_update_disksize(handle, inode, end,
|
||||
is_disksize_grow);
|
||||
if (ret)
|
||||
goto out_journal;
|
||||
}
|
||||
|
||||
if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN)
|
||||
ret = ext4_convert_unwritten_extents(handle, inode, pos,
|
||||
size, NULL);
|
||||
|
||||
out_journal:
|
||||
err = ext4_journal_stop(handle);
|
||||
if (!ret)
|
||||
ret = err;
|
||||
out_err:
|
||||
if (ret < 0 && !ext4_emergency_state(sb)) {
|
||||
ext4_msg(sb, KERN_EMERG,
|
||||
"failed to convert unwritten extents to written extents or update inode size -- potential data loss! (inode %llu, error %d)",
|
||||
inode->i_ino, ret);
|
||||
}
|
||||
out:
|
||||
iomap_finish_ioends(ioend, ret);
|
||||
}
|
||||
|
||||
/*
|
||||
* Work on buffered iomap completed IO, to convert unwritten extents to
|
||||
* mapped extents
|
||||
*/
|
||||
void ext4_iomap_end_io(struct work_struct *work)
|
||||
{
|
||||
struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
|
||||
i_rsv_conversion_work);
|
||||
struct iomap_ioend *ioend;
|
||||
struct list_head ioend_list;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
|
||||
list_replace_init(&ei->i_rsv_conversion_list, &ioend_list);
|
||||
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
|
||||
|
||||
iomap_sort_ioends(&ioend_list);
|
||||
while (!list_empty(&ioend_list)) {
|
||||
ioend = list_entry(ioend_list.next, struct iomap_ioend, io_list);
|
||||
list_del_init(&ioend->io_list);
|
||||
iomap_ioend_try_merge(ioend, &ioend_list);
|
||||
ext4_iomap_finish_ioend(ioend);
|
||||
}
|
||||
}
|
||||
|
||||
void ext4_iomap_end_bio(struct bio *bio)
|
||||
{
|
||||
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
|
||||
struct ext4_inode_info *ei = EXT4_I(ioend->io_inode);
|
||||
unsigned long io_mode = (unsigned long)ioend->io_private;
|
||||
unsigned long flags;
|
||||
|
||||
/*
|
||||
* This is a disksize-pending I/O: clear the disksize-pending
|
||||
* state set in ext4_block_zero_eof() and wake up all waiters
|
||||
* that will update the inode i_disksize.
|
||||
*/
|
||||
if (io_mode == EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO)
|
||||
ext4_iomap_clear_disksize_pending(ioend->io_inode);
|
||||
|
||||
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
|
||||
if (list_empty(&ei->i_rsv_conversion_list))
|
||||
queue_work(EXT4_SB(ioend->io_inode->i_sb)->rsv_conversion_wq,
|
||||
&ei->i_rsv_conversion_work);
|
||||
list_add_tail(&ioend->io_list, &ei->i_rsv_conversion_list);
|
||||
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1462,11 +1462,11 @@ static void ext4_update_super(struct super_block *sb,
|
|||
*
|
||||
* The precise rules we use are:
|
||||
*
|
||||
* * Writers must perform a smp_wmb() after updating all
|
||||
* dependent data and before modifying the groups count
|
||||
* * Writers must use a release store when updating the groups count
|
||||
* after all dependent data has been updated
|
||||
*
|
||||
* * Readers must perform an smp_rmb() after reading the groups
|
||||
* count and before reading any dependent data.
|
||||
* * Readers must use an acquire load when reading the groups
|
||||
* count before reading any dependent data.
|
||||
*
|
||||
* NB. These rules can be relaxed when checking the group count
|
||||
* while freeing data, as we can only allocate from a block
|
||||
|
|
@ -1474,12 +1474,15 @@ static void ext4_update_super(struct super_block *sb,
|
|||
* only then free after serialising in turn against that
|
||||
* allocation.
|
||||
*/
|
||||
smp_wmb();
|
||||
|
||||
/* Update the global fs size fields */
|
||||
sbi->s_groups_count += flex_gd->count;
|
||||
sbi->s_blockfile_groups = min(sbi->s_groups_count,
|
||||
(EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb)));
|
||||
/* Pairs with smp_load_acquire() in ext4_get_groups_count() */
|
||||
smp_store_release(&sbi->s_groups_count,
|
||||
sbi->s_groups_count + flex_gd->count);
|
||||
/* Pairs with smp_load_acquire() in ext4_get_allocation_groups_count() */
|
||||
smp_store_release(&sbi->s_blockfile_groups,
|
||||
min(sbi->s_groups_count,
|
||||
(EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))));
|
||||
|
||||
/* Update the reserved block counts only once the new group is
|
||||
* active. */
|
||||
|
|
|
|||
|
|
@ -100,13 +100,21 @@ static const struct fs_parameter_spec ext4_param_specs[];
|
|||
* Lock ordering
|
||||
*
|
||||
* page fault path:
|
||||
* mmap_lock -> sb_start_pagefault -> invalidate_lock (r) -> transaction start
|
||||
* -> page lock -> i_data_sem (rw)
|
||||
* - buffer_head path:
|
||||
* mmap_lock -> sb_start_pagefault -> invalidate_lock (r) ->
|
||||
* transaction start -> folio lock -> i_data_sem (rw)
|
||||
* - iomap path:
|
||||
* mmap_lock -> sb_start_pagefault -> invalidate_lock (r) ->
|
||||
* folio lock -> transaction start -> i_data_sem (rw)
|
||||
*
|
||||
* buffered write path:
|
||||
* sb_start_write -> i_mutex -> mmap_lock
|
||||
* sb_start_write -> i_mutex -> transaction start -> page lock ->
|
||||
* i_data_sem (rw)
|
||||
* sb_start_write -> i_rwsem (w) -> mmap_lock
|
||||
* - buffer_head path:
|
||||
* sb_start_write -> i_rwsem (w) -> transaction start -> folio lock ->
|
||||
* i_data_sem (rw)
|
||||
* - iomap path:
|
||||
* sb_start_write -> i_rwsem (w) -> transaction start -> i_data_sem (rw)
|
||||
* sb_start_write -> i_rwsem (w) -> folio lock (not under an active handle)
|
||||
*
|
||||
* truncate:
|
||||
* sb_start_write -> i_mutex -> invalidate_lock (w) -> i_mmap_rwsem (w) ->
|
||||
|
|
@ -119,7 +127,10 @@ static const struct fs_parameter_spec ext4_param_specs[];
|
|||
* sb_start_write -> i_mutex -> transaction start -> i_data_sem (rw)
|
||||
*
|
||||
* writepages:
|
||||
* transaction start -> page lock(s) -> i_data_sem (rw)
|
||||
* - buffer_head path:
|
||||
* transaction start -> folio lock(s) -> i_data_sem (rw)
|
||||
* - iomap path:
|
||||
* folio lock -> transaction start -> i_data_sem (rw)
|
||||
*/
|
||||
|
||||
static const struct fs_context_operations ext4_context_ops = {
|
||||
|
|
@ -1494,6 +1505,14 @@ static void ext4_destroy_inode(struct inode *inode)
|
|||
"Inode %llu (%p): i_reserved_data_blocks (%u) not cleared!",
|
||||
inode->i_ino, EXT4_I(inode),
|
||||
EXT4_I(inode)->i_reserved_data_blocks);
|
||||
|
||||
if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ERROR_FS) &&
|
||||
!ext4_emergency_state(inode->i_sb) &&
|
||||
WARN_ON_ONCE(ext4_test_inode_state(inode,
|
||||
EXT4_STATE_DISKSIZE_GROW_PENDING)))
|
||||
ext4_msg(inode->i_sb, KERN_ERR,
|
||||
"Inode %llu (%p): EXT4_STATE_DISKSIZE_GROW_PENDING not cleared!",
|
||||
inode->i_ino, EXT4_I(inode));
|
||||
}
|
||||
|
||||
static void ext4_shutdown(struct super_block *sb)
|
||||
|
|
@ -1736,6 +1755,7 @@ enum {
|
|||
Opt_discard, Opt_nodiscard, Opt_init_itable, Opt_noinit_itable,
|
||||
Opt_max_dir_size_kb, Opt_nojournal_checksum, Opt_nombcache,
|
||||
Opt_no_prefetch_block_bitmaps, Opt_mb_optimize_scan,
|
||||
Opt_buffered_iomap, Opt_nobuffered_iomap,
|
||||
Opt_errors, Opt_data, Opt_data_err, Opt_jqfmt, Opt_dax_type,
|
||||
#ifdef CONFIG_EXT4_DEBUG
|
||||
Opt_fc_debug_max_replay, Opt_fc_debug_force
|
||||
|
|
@ -1874,6 +1894,8 @@ static const struct fs_parameter_spec ext4_param_specs[] = {
|
|||
fsparam_flag ("no_prefetch_block_bitmaps",
|
||||
Opt_no_prefetch_block_bitmaps),
|
||||
fsparam_s32 ("mb_optimize_scan", Opt_mb_optimize_scan),
|
||||
fsparam_flag ("buffered_iomap", Opt_buffered_iomap),
|
||||
fsparam_flag ("nobuffered_iomap", Opt_nobuffered_iomap),
|
||||
fsparam_string ("check", Opt_removed), /* mount option from ext2/3 */
|
||||
fsparam_flag ("nocheck", Opt_removed), /* mount option from ext2/3 */
|
||||
fsparam_flag ("reservation", Opt_removed), /* mount option from ext2/3 */
|
||||
|
|
@ -1967,6 +1989,10 @@ static const struct mount_opts {
|
|||
{Opt_nombcache, EXT4_MOUNT_NO_MBCACHE, MOPT_SET},
|
||||
{Opt_no_prefetch_block_bitmaps, EXT4_MOUNT_NO_PREFETCH_BLOCK_BITMAPS,
|
||||
MOPT_SET},
|
||||
{Opt_buffered_iomap, EXT4_MOUNT2_BUFFERED_IOMAP,
|
||||
MOPT_SET | MOPT_2 | MOPT_EXT4_ONLY},
|
||||
{Opt_nobuffered_iomap, EXT4_MOUNT2_BUFFERED_IOMAP,
|
||||
MOPT_CLEAR | MOPT_2 | MOPT_EXT4_ONLY},
|
||||
#ifdef CONFIG_EXT4_DEBUG
|
||||
{Opt_fc_debug_force, EXT4_MOUNT2_JOURNAL_FAST_COMMIT,
|
||||
MOPT_SET | MOPT_2 | MOPT_EXT4_ONLY},
|
||||
|
|
@ -2840,6 +2866,14 @@ static int ext4_check_opt_consistency(struct fs_context *fc,
|
|||
}
|
||||
|
||||
if (is_remount) {
|
||||
if (test_opt(sb, DELALLOC) &&
|
||||
(ctx->mask_s_mount_opt & EXT4_MOUNT_DELALLOC) &&
|
||||
!ctx_test_mount_opt(ctx, EXT4_MOUNT_DELALLOC)) {
|
||||
ext4_msg(sb, KERN_ERR,
|
||||
"can't disable delalloc during remount");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!sbi->s_journal &&
|
||||
ctx_test_mount_opt(ctx, EXT4_MOUNT_DATA_ERR_ABORT)) {
|
||||
ext4_msg(NULL, KERN_WARNING,
|
||||
|
|
@ -6133,6 +6167,7 @@ static int ext4_load_journal(struct super_block *sb,
|
|||
int err = 0;
|
||||
int really_read_only;
|
||||
int journal_dev_ro;
|
||||
bool enable_write = false;
|
||||
|
||||
if (WARN_ON_ONCE(!ext4_has_feature_journal(sb)))
|
||||
return -EFSCORRUPTED;
|
||||
|
|
@ -6189,6 +6224,7 @@ static int ext4_load_journal(struct super_block *sb,
|
|||
}
|
||||
ext4_msg(sb, KERN_INFO, "write access will "
|
||||
"be enabled during recovery");
|
||||
enable_write = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6205,7 +6241,19 @@ static int ext4_load_journal(struct super_block *sb,
|
|||
if (save)
|
||||
memcpy(save, ((char *) es) +
|
||||
EXT4_S_ERR_START, EXT4_S_ERR_LEN);
|
||||
/*
|
||||
* Fast commit replay performs regular ext4 metadata updates
|
||||
* (see ext4_fc_replay()) which refuse to run on a read-only
|
||||
* superblock. We promised write access above, so make that
|
||||
* true for the duration of the recovery, the same way
|
||||
* ext4_orphan_cleanup() does. The superblock is not published
|
||||
* yet, so nothing can observe the transient state.
|
||||
*/
|
||||
if (enable_write)
|
||||
sb->s_flags &= ~SB_RDONLY;
|
||||
err = jbd2_journal_load(journal);
|
||||
if (enable_write)
|
||||
sb->s_flags |= SB_RDONLY;
|
||||
if (save && memcmp(((char *) es) + EXT4_S_ERR_START,
|
||||
save, EXT4_S_ERR_LEN)) {
|
||||
memcpy(((char *) es) + EXT4_S_ERR_START,
|
||||
|
|
@ -6698,13 +6746,6 @@ static int __ext4_remount(struct fs_context *fc, struct super_block *sb)
|
|||
goto restore_opts;
|
||||
}
|
||||
|
||||
if ((old_opts.s_mount_opt & EXT4_MOUNT_DELALLOC) &&
|
||||
!test_opt(sb, DELALLOC)) {
|
||||
ext4_msg(sb, KERN_ERR, "can't disable delalloc during remount");
|
||||
err = -EINVAL;
|
||||
goto restore_opts;
|
||||
}
|
||||
|
||||
sb->s_flags = (sb->s_flags & ~SB_POSIXACL) |
|
||||
(test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0);
|
||||
|
||||
|
|
|
|||
|
|
@ -848,7 +848,6 @@ int jbd2_fc_get_buf(journal_t *journal, struct buffer_head **bh_out)
|
|||
|
||||
fc_off = journal->j_fc_off;
|
||||
blocknr = journal->j_fc_first + fc_off;
|
||||
journal->j_fc_off++;
|
||||
ret = jbd2_journal_bmap(journal, blocknr, &pblock);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
|
@ -858,6 +857,7 @@ int jbd2_fc_get_buf(journal_t *journal, struct buffer_head **bh_out)
|
|||
return -ENOMEM;
|
||||
|
||||
journal->j_fc_wbuf[fc_off] = bh;
|
||||
journal->j_fc_off++;
|
||||
|
||||
*bh_out = bh;
|
||||
|
||||
|
|
|
|||
|
|
@ -795,7 +795,7 @@ struct journal_s
|
|||
/**
|
||||
* @j_state_lock: Protect the various scalars in the journal.
|
||||
*/
|
||||
rwlock_t j_state_lock;
|
||||
rwlock_t j_state_lock ____cacheline_aligned_in_smp;
|
||||
|
||||
/**
|
||||
* @j_barrier_count:
|
||||
|
|
|
|||
|
|
@ -3157,6 +3157,215 @@ TRACE_EVENT(ext4_move_extent_exit,
|
|||
__entry->ret)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(ext4_set_iomap_class,
|
||||
TP_PROTO(struct inode *inode, struct ext4_map_blocks *map,
|
||||
loff_t offset, loff_t length, unsigned int flags),
|
||||
TP_ARGS(inode, map, offset, length, flags),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(ext4_lblk_t, m_lblk)
|
||||
__field(ext4_fsblk_t, pblk)
|
||||
__field(unsigned int, m_len)
|
||||
__field(unsigned int, m_flags)
|
||||
__field(u64, m_seq)
|
||||
__field(loff_t, offset)
|
||||
__field(loff_t, length)
|
||||
__field(unsigned int, iomap_flags)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->m_lblk = map->m_lblk;
|
||||
__entry->pblk = map->m_pblk;
|
||||
__entry->m_len = map->m_len;
|
||||
__entry->m_flags = map->m_flags;
|
||||
__entry->m_seq = map->m_seq;
|
||||
__entry->offset = offset;
|
||||
__entry->length = length;
|
||||
__entry->iomap_flags = flags;
|
||||
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu m_lblk %u pblk %llu m_len %u m_flags %s m_seq %llu orig_off 0x%llx orig_len 0x%llx iomap_flags 0x%x",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev),
|
||||
__entry->ino, __entry->m_lblk, __entry->pblk, __entry->m_len,
|
||||
show_mflags(__entry->m_flags), __entry->m_seq,
|
||||
__entry->offset, __entry->length, __entry->iomap_flags)
|
||||
)
|
||||
|
||||
#define DEFINE_SET_IOMAP_EVENT(name) \
|
||||
DEFINE_EVENT(ext4_set_iomap_class, name, \
|
||||
TP_PROTO(struct inode *inode, struct ext4_map_blocks *map, \
|
||||
loff_t offset, loff_t length, unsigned int flags), \
|
||||
TP_ARGS(inode, map, offset, length, flags))
|
||||
|
||||
DEFINE_SET_IOMAP_EVENT(ext4_iomap_buffered_read_begin);
|
||||
DEFINE_SET_IOMAP_EVENT(ext4_iomap_buffered_write_begin);
|
||||
DEFINE_SET_IOMAP_EVENT(ext4_iomap_map_writeback_range);
|
||||
DEFINE_SET_IOMAP_EVENT(ext4_iomap_zero_begin);
|
||||
|
||||
DECLARE_EVENT_CLASS(ext4_iomap_disksize_pending,
|
||||
TP_PROTO(struct inode *inode),
|
||||
TP_ARGS(inode),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(loff_t, i_disksize)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->i_disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu i_disksize %lld",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev),
|
||||
__entry->ino, __entry->i_disksize)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(ext4_iomap_disksize_pending, ext4_iomap_mark_disksize_pending,
|
||||
TP_PROTO(struct inode *inode),
|
||||
TP_ARGS(inode)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(ext4_iomap_disksize_pending, ext4_iomap_clear_disksize_pending,
|
||||
TP_PROTO(struct inode *inode),
|
||||
TP_ARGS(inode)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(ext4_iomap_disksize_pending, ext4_iomap_wait_disksize_pending,
|
||||
TP_PROTO(struct inode *inode),
|
||||
TP_ARGS(inode)
|
||||
);
|
||||
|
||||
/* disksize pending I/O tracepoints for iomap Buffered I/O path */
|
||||
DECLARE_EVENT_CLASS(ext4_iomap_wb_disksize_pending,
|
||||
TP_PROTO(struct inode *inode, loff_t io_offset, size_t io_size),
|
||||
TP_ARGS(inode, io_offset, io_size),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(loff_t, io_offset)
|
||||
__field(size_t, io_size)
|
||||
__field(loff_t, i_size)
|
||||
__field(loff_t, i_disksize)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->io_offset = io_offset;
|
||||
__entry->io_size = io_size;
|
||||
__entry->i_size = i_size_read(inode);
|
||||
__entry->i_disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu io_offset %lld io_size %zu i_size %lld i_disksize %lld",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev),
|
||||
__entry->ino, __entry->io_offset, __entry->io_size,
|
||||
__entry->i_size, __entry->i_disksize)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(ext4_iomap_wb_disksize_pending,
|
||||
ext4_iomap_wb_disksize_pending_submit,
|
||||
TP_PROTO(struct inode *inode, loff_t io_offset, size_t io_size),
|
||||
TP_ARGS(inode, io_offset, io_size)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(ext4_iomap_wb_disksize_pending,
|
||||
ext4_iomap_wb_disksize_pending_wait,
|
||||
TP_PROTO(struct inode *inode, loff_t io_offset, size_t io_size),
|
||||
TP_ARGS(inode, io_offset, io_size)
|
||||
);
|
||||
|
||||
TRACE_EVENT(ext4_iomap_wb_disksize_pending_complete,
|
||||
TP_PROTO(struct inode *inode, loff_t io_offset, size_t io_size,
|
||||
int ret),
|
||||
TP_ARGS(inode, io_offset, io_size, ret),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(loff_t, io_offset)
|
||||
__field(size_t, io_size)
|
||||
__field(loff_t, i_size)
|
||||
__field(loff_t, i_disksize)
|
||||
__field(int, ret)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->io_offset = io_offset;
|
||||
__entry->io_size = io_size;
|
||||
__entry->i_size = i_size_read(inode);
|
||||
__entry->i_disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
|
||||
__entry->ret = ret;
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu io_offset %lld io_size %zu ret %d i_size %lld i_disksize %lld",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev),
|
||||
__entry->ino, __entry->io_offset, __entry->io_size,
|
||||
__entry->ret, __entry->i_size, __entry->i_disksize)
|
||||
);
|
||||
|
||||
/* i_disksize update tracepoint */
|
||||
TRACE_EVENT(ext4_iomap_wb_update_disksize,
|
||||
TP_PROTO(struct inode *inode, loff_t end, loff_t i_size,
|
||||
loff_t i_disksize, loff_t new_disksize,
|
||||
bool is_disksize_grow),
|
||||
TP_ARGS(inode, end, i_size, i_disksize, new_disksize,
|
||||
is_disksize_grow),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(loff_t, end)
|
||||
__field(loff_t, i_size)
|
||||
__field(loff_t, i_disksize)
|
||||
__field(loff_t, new_disksize)
|
||||
__field(bool, is_disksize_grow)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->end = end;
|
||||
__entry->i_size = i_size;
|
||||
__entry->i_disksize = i_disksize;
|
||||
__entry->new_disksize = new_disksize;
|
||||
__entry->is_disksize_grow = is_disksize_grow;
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu end %lld i_size %lld i_disksize %lld new_disksize %lld is_disksize_grow %d",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev),
|
||||
__entry->ino, __entry->end, __entry->i_size,
|
||||
__entry->i_disksize, __entry->new_disksize,
|
||||
__entry->is_disksize_grow)
|
||||
);
|
||||
|
||||
/* Block zero EOF tracepoint */
|
||||
TRACE_EVENT(ext4_block_zero_eof,
|
||||
TP_PROTO(struct inode *inode, loff_t from, loff_t length,
|
||||
bool did_zero, bool zero_written),
|
||||
TP_ARGS(inode, from, length, did_zero, zero_written),
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev)
|
||||
__field(u64, ino)
|
||||
__field(loff_t, from)
|
||||
__field(loff_t, length)
|
||||
__field(loff_t, i_size)
|
||||
__field(loff_t, i_disksize)
|
||||
__field(bool, did_zero)
|
||||
__field(bool, zero_written)
|
||||
),
|
||||
TP_fast_assign(
|
||||
__entry->dev = inode->i_sb->s_dev;
|
||||
__entry->ino = inode->i_ino;
|
||||
__entry->from = from;
|
||||
__entry->length = length;
|
||||
__entry->i_size = inode->i_size;
|
||||
__entry->i_disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
|
||||
__entry->did_zero = did_zero;
|
||||
__entry->zero_written = zero_written;
|
||||
),
|
||||
TP_printk("dev %d:%d ino %llu zero EOF from %lld length %lld i_size %lld i_disksize %lld did_zero %d zero_written %d",
|
||||
MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino,
|
||||
__entry->from, __entry->length, __entry->i_size,
|
||||
__entry->i_disksize, __entry->did_zero, __entry->zero_written)
|
||||
);
|
||||
|
||||
#endif /* _TRACE_EXT4_H */
|
||||
|
||||
/* This part must be outside protection */
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user