diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 0cebc89799dd..e7e968c6df92 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -271,7 +271,7 @@ Description: Shows all enabled kernel features. inode_crtime, lost_found, verity, sb_checksum, casefold, readonly, compression, test_dummy_encryption_v2, atomic_write, pin_file, encrypted_casefold, linear_lookup, - fserror. + fserror, metadata_cache, resizable_tail. What: /sys/fs/f2fs//inject_rate Date: May 2016 @@ -1014,9 +1014,48 @@ Description: Every time a write operation completes f2fs_write_end_io() is (atc) context. The default value for this attribute is UINT_MAX which means that this functionality is disabled by default. +What: /sys/fs/f2fs//max_atc_write_bio_entry_cnt +Date: October 2026 +Contact: Chao Yu +Description: Every time a write operation completes f2fs_write_end_io() is + called. This function may be called from an atomic context, + e.g. from inside an interrupt handler. This attribute controls + the maximum number of entries (folios or cached blocks) in a + write bio that is completed in atomic (atc) context. The default + value for this attribute is UINT_MAX which means that this + functionality is disabled by default. + What: /sys/fs/f2fs//pinned_area_max_secno Date: August 2026 Contact: "Daeho Jeong" Description: This is a read-only entry to show the upper bound section number for pinned files. Pinned files will only be allocated within sections 0 to pinned_area_max_secno - 1. + +What: /sys/fs/f2fs//cache_wb_interval +Date: August 2026 +Contact: "Chao Yu" +Description: This is a writable entry to control writeback interval of + f2fs_writeback-x:y, the range is [100, 30000], by default the value + is 5000, unit is ms. + +What: /sys/fs/f2fs//cache_wb_dirty_threshold +Date: October 2026 +Contact: "Chao Yu" +Description: This is a writable entry for metadata cache. f2fs_writeback-x:y + will be woken up for writeback once the number of dirty meta and + node caches reaches this value. 0 disables this check. By default + the value is 8192, unit is cache entries. + +What: /sys/fs/f2fs//cache_wb_total_threshold +Date: October 2026 +Contact: "Chao Yu" +Description: This is a writable entry for metadata cache. f2fs_writeback-x:y + will be woken up for writeback only if the total number of cached + entries also reaches this value. 0 disables this check. By default + the value is 0, unit is cache entries. + + If both cache_wb_dirty_threshold and cache_wb_total_threshold are + 0, f2fs_writeback-x:y only runs every cache_wb_interval, unless + checkpoint is disabled, in which case it will still be woken up + once there are enough dirty caches to be gathered for writeback. diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index 771216f45207..55798421f2c4 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -421,7 +421,8 @@ resizable_tail_secno=%u Control the number of sections at the tail of the (MAIN_SECS - resizable_tail_secno) - 1. If set to 0 (default), there is no tail restriction unless running on a zoned block device where conventional zones are - used. + used. Online shrink consumes this reservation, reducing + the value by the number of removed sections, down to zero. ======================== ============================================================ Debugfs Entries diff --git a/MAINTAINERS b/MAINTAINERS index 03649f08c7c7..1ed6d81ed2f6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10169,7 +10169,7 @@ S: Maintained W: https://f2fs.wiki.kernel.org/ Q: https://patchwork.kernel.org/project/f2fs/list/ B: https://bugzilla.kernel.org/enter_bug.cgi?product=File%20System&component=f2fs -T: git git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git dev F: Documentation/ABI/testing/sysfs-fs-f2fs F: Documentation/filesystems/f2fs.rst F: fs/f2fs/ diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile index 8a7322d229e4..fbf49c30b066 100644 --- a/fs/f2fs/Makefile +++ b/fs/f2fs/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_F2FS_FS) += f2fs.o f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o -f2fs-y += shrinker.o extent_cache.o sysfs.o +f2fs-y += shrinker.o extent_cache.o sysfs.o cache.o f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 22ca39e0aad6..a7485bc38252 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -181,7 +181,7 @@ static void *f2fs_acl_to_disk(struct f2fs_sb_info *sbi, } static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type, - struct folio *dfolio) + struct f2fs_cached_block *entry) { int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; void *value = NULL; @@ -191,13 +191,13 @@ static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type, if (type == ACL_TYPE_ACCESS) name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; - retval = f2fs_getxattr(inode, name_index, "", NULL, 0, dfolio); + retval = f2fs_getxattr(inode, name_index, "", NULL, 0, entry); if (retval > 0) { value = f2fs_kmalloc(F2FS_I_SB(inode), retval, GFP_F2FS_ZERO); if (!value) return ERR_PTR(-ENOMEM); retval = f2fs_getxattr(inode, name_index, "", value, - retval, dfolio); + retval, entry); } if (retval > 0) @@ -242,7 +242,7 @@ static int f2fs_acl_update_mode(const struct mnt_idmap *idmap, static int __f2fs_set_acl(const struct mnt_idmap *idmap, struct inode *inode, int type, - struct posix_acl *acl, struct folio *ifolio) + struct posix_acl *acl, struct f2fs_cached_block *ientry) { int name_index; void *value = NULL; @@ -253,7 +253,7 @@ static int __f2fs_set_acl(const struct mnt_idmap *idmap, switch (type) { case ACL_TYPE_ACCESS: name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; - if (acl && !ifolio) { + if (acl && !ientry) { error = f2fs_acl_update_mode(idmap, inode, &mode, &acl); if (error) return error; @@ -279,7 +279,7 @@ static int __f2fs_set_acl(const struct mnt_idmap *idmap, } } - error = f2fs_setxattr(inode, name_index, "", value, size, ifolio, 0); + error = f2fs_setxattr(inode, name_index, "", value, size, ientry, 0); kfree(value); if (!error) @@ -374,7 +374,7 @@ static int f2fs_acl_create_masq(struct posix_acl *acl, umode_t *mode_p) static int f2fs_acl_create(struct inode *dir, umode_t *mode, struct posix_acl **default_acl, struct posix_acl **acl, - struct folio *dfolio) + struct f2fs_cached_block *entry) { struct posix_acl *p; struct posix_acl *clone; @@ -386,7 +386,7 @@ static int f2fs_acl_create(struct inode *dir, umode_t *mode, if (S_ISLNK(*mode) || !IS_POSIXACL(dir)) return 0; - p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, dfolio); + p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, entry); if (!p || p == ERR_PTR(-EOPNOTSUPP)) { *mode &= ~current_umask(); return 0; @@ -423,13 +423,13 @@ static int f2fs_acl_create(struct inode *dir, umode_t *mode, return ret; } -int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio, - struct folio *dfolio) +int f2fs_init_acl(struct inode *inode, struct inode *dir, struct f2fs_cached_block *ientry, + struct f2fs_cached_block *dentry) { struct posix_acl *default_acl = NULL, *acl = NULL; int error; - error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dfolio); + error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dentry); if (error) return error; @@ -437,7 +437,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio, if (default_acl) { error = __f2fs_set_acl(NULL, inode, ACL_TYPE_DEFAULT, - default_acl, ifolio); + default_acl, ientry); posix_acl_release(default_acl); } else { inode->i_default_acl = NULL; @@ -445,7 +445,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio, if (acl) { if (!error) error = __f2fs_set_acl(NULL, inode, ACL_TYPE_ACCESS, - acl, ifolio); + acl, ientry); posix_acl_release(acl); } else { inode->i_acl = NULL; diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h index 01ec74b6daf2..b1085efcc05e 100644 --- a/fs/f2fs/acl.h +++ b/fs/f2fs/acl.h @@ -36,14 +36,16 @@ struct f2fs_acl_header { struct posix_acl *f2fs_get_acl(struct inode *, int, bool); int f2fs_set_acl(const struct mnt_idmap *, struct dentry *, struct posix_acl *, int); -int f2fs_init_acl(struct inode *, struct inode *, struct folio *ifolio, - struct folio *dfolio); +int f2fs_init_acl(struct inode *inode, struct inode *dir, + struct f2fs_cached_block *ientry, + struct f2fs_cached_block *dentry); #else #define f2fs_get_acl NULL #define f2fs_set_acl NULL static inline int f2fs_init_acl(struct inode *inode, struct inode *dir, - struct folio *ifolio, struct folio *dfolio) + struct f2fs_cached_block *ientry, + struct f2fs_cached_block *dentry) { return 0; } diff --git a/fs/f2fs/cache.c b/fs/f2fs/cache.c new file mode 100644 index 000000000000..ddf94b5791e7 --- /dev/null +++ b/fs/f2fs/cache.c @@ -0,0 +1,894 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Google LLC + * Author: Chao Yu + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "f2fs.h" +#include "cache.h" +#include "node.h" +#include +#include "segment.h" + +static void f2fs_free_cache(struct f2fs_cached_block *entry); + +void f2fs_cache_wait_writeback_cond(struct f2fs_cached_block *entry, + enum page_type type) +{ + /* in case the entry was truncated or on-going shrink */ + if (!entry->cache) + return; + + if (!f2fs_cache_test_writeback(entry)) + return; + + /* submit cached bio */ + f2fs_submit_merged_write_cache(entry->cache->sbi, entry, 0, type); + + wait_on_bit_io(&entry->state, F2FS_BLOCK_WRITEBACK, + TASK_UNINTERRUPTIBLE); +} + +void f2fs_cache_wait_writeback(struct f2fs_cached_block *entry) +{ + /* in case the entry was truncated or on-going shrink */ + if (!entry->cache) + return; + + f2fs_cache_wait_writeback_cond(entry, + IS_META_CACHE(entry->cache) ? META : NODE); +} + +void f2fs_cache_update_tag(struct f2fs_cached_block *entry, + unsigned int clear_from, unsigned int set_to) +{ + struct f2fs_cached_block_list *cache = entry->cache; + unsigned long flags; + + spin_lock_irqsave(&cache->tree_lock, flags); + if (clear_from != F2FS_CACHE_TAG_NONE) + radix_tree_tag_clear(&cache->root, entry->index, clear_from); + if (set_to != F2FS_CACHE_TAG_NONE) + radix_tree_tag_set(&cache->root, entry->index, set_to); + spin_unlock_irqrestore(&cache->tree_lock, flags); +} + +bool f2fs_mark_cache_dirty(struct f2fs_cached_block *entry) +{ + struct f2fs_cached_block_list *cache = entry->cache; + + f2fs_cache_set_uptodate(entry); + +#ifdef CONFIG_F2FS_CHECK_FS + if (f2fs_is_node_cache(entry) && IS_INODE(cache->sbi, entry)) + f2fs_inode_chksum_set(cache->sbi, entry); +#endif + + if (!f2fs_cache_test_and_set_dirty(entry)) { + enum count_type type = IS_META_CACHE(cache) ? + F2FS_DIRTY_META : F2FS_DIRTY_NODES; + + trace_f2fs_cache_set_dirty(entry, + IS_META_CACHE(cache) ? META : NODE); + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_NONE, + F2FS_CACHE_TAG_DIRTY); + inc_cache_count(cache->sbi, type); + f2fs_wake_up_cache_wb(cache->sbi); + return true; + } + + return false; +} + +void f2fs_drop_cache_dirty(struct f2fs_cached_block *entry) +{ + + struct f2fs_cached_block_list *cache = entry->cache; + enum count_type type = IS_META_CACHE(cache) ? + F2FS_DIRTY_META : F2FS_DIRTY_NODES; + + f2fs_cache_clear_uptodate(entry); + + if (!f2fs_cache_test_and_clear_dirty(entry)) + return; + + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, + F2FS_CACHE_TAG_NONE); + dec_cache_count(cache->sbi, type); +} + +void f2fs_start_cache_writeback(struct f2fs_cached_block *entry) +{ + f2fs_cache_set_writeback(entry); + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, + F2FS_CACHE_TAG_WRITEBACK); +} + +/* + * The temporary references taken around clear_and_wake_up_bit() are counted + * in units of F2FS_CACHE_PIN_BIAS, like GUP_PIN_COUNTING_BIAS for folios, so + * that they can be told apart from the regular references. + */ +#define F2FS_CACHE_PIN_BIAS (1 << 16) + +static void f2fs_cache_pin(struct f2fs_cached_block *entry) +{ + atomic_add(F2FS_CACHE_PIN_BIAS, &entry->refcount); +} + +static void f2fs_cache_unpin(struct f2fs_cached_block *entry) +{ + int ref = atomic_sub_return(F2FS_CACHE_PIN_BIAS, &entry->refcount); + + WARN_ON_ONCE(ref < 0); + if (!ref) + f2fs_free_cache(entry); +} + +void f2fs_end_cache_writeback(struct f2fs_cached_block *entry) +{ + /* + * should call f2fs_cache_update_tag() before clearing writeback bit, + * in case f2fs_truncate_cache() set entry->cache to NULL. + */ + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_WRITEBACK, + F2FS_CACHE_TAG_NONE); + /* + * Writeback does not hold an entry reference of its own, relying + * on truncation to wait for the clearing of F2FS_BLOCK_WRITEBACK. + * But here we must make sure that the entry is not freed and + * reused before clear_and_wake_up_bit(). + */ + f2fs_cache_pin(entry); + clear_and_wake_up_bit(F2FS_BLOCK_WRITEBACK, &entry->state); + f2fs_cache_unpin(entry); +} + +static int f2fs_cache_refcount(struct f2fs_cached_block *entry) +{ + return atomic_read(&entry->refcount); +} + +/* the number of regular references, excluding the temporary ones */ +static int f2fs_cache_users(struct f2fs_cached_block *entry) +{ + return f2fs_cache_refcount(entry) & (F2FS_CACHE_PIN_BIAS - 1); +} + +static void f2fs_do_free_cache(struct f2fs_cached_block *entry) +{ + kfree(entry->data); + kfree(entry); +} + +static void f2fs_free_cache(struct f2fs_cached_block *entry) +{ + WARN_ON_ONCE(!list_empty(&entry->list)); + WARN_ON_ONCE(f2fs_cache_refcount(entry)); + f2fs_do_free_cache(entry); +} + +void f2fs_cache_get(struct f2fs_cached_block *entry) +{ + atomic_inc(&entry->refcount); +} + +static bool f2fs_cache_put(struct f2fs_cached_block *entry) +{ + WARN_ON_ONCE(!f2fs_cache_refcount(entry)); + if (atomic_dec_and_test(&entry->refcount)) { + f2fs_free_cache(entry); + return true; + } + return false; +} + +static struct f2fs_cached_block *f2fs_create_cache( + struct f2fs_cached_block_list *cache, + unsigned long index, bool nofail) +{ + struct f2fs_sb_info *sbi = cache->sbi; + struct f2fs_cached_block *entry; + unsigned int flags = GFP_NOFS; + + if (index == ULONG_MAX) + return ERR_PTR(-ERANGE); + + if (nofail) { + flags |= __GFP_NOFAIL; + entry = kzalloc_obj(*entry, flags); + entry->data = kmalloc(sbi->blocksize, flags); + } else { + entry = f2fs_kzalloc(sbi, sizeof(*entry), flags); + if (!entry) + return ERR_PTR(-ENOMEM); + + entry->data = f2fs_kmalloc(sbi, sbi->blocksize, flags); + if (!entry->data) { + kfree(entry); + return ERR_PTR(-ENOMEM); + } + } + entry->index = index; + + atomic_set(&entry->refcount, 0); + if (!IS_COMPRESS_CACHE(cache)) + entry->next_entry = NULL; + else + entry->ino = 0; + INIT_LIST_HEAD(&entry->list); + + entry->cache = cache; + + return entry; +} + +static struct f2fs_cached_block *f2fs_insert_cache( + struct f2fs_cached_block_list *cache, + unsigned long index, + struct f2fs_cached_block *new) +{ + struct f2fs_cached_block *e; + int ret; + unsigned long flags; + + ret = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); + f2fs_bug_on(cache->sbi, ret); + + spin_lock(&cache->list_lock); + spin_lock_irqsave(&cache->tree_lock, flags); + e = radix_tree_lookup(&cache->root, index); + if (!e) { + e = new; + f2fs_bug_on(cache->sbi, f2fs_cache_refcount(e)); + + ret = radix_tree_insert(&cache->root, index, e); + f2fs_bug_on(cache->sbi, ret); + + /* radix tree referenced cache entry */ + f2fs_cache_get(e); + f2fs_bug_on(cache->sbi, !list_empty(&e->list)); + list_add_tail(&e->list, &cache->lru_list); + cache->num_entries++; + } + f2fs_cache_get(e); + spin_unlock_irqrestore(&cache->tree_lock, flags); + spin_unlock(&cache->list_lock); + radix_tree_preload_end(); + + if (new != e) { + f2fs_bug_on(cache->sbi, f2fs_cache_refcount(new)); + f2fs_do_free_cache(new); + } + + return e; +} + +struct f2fs_cached_block *f2fs_find_cache( + struct f2fs_cached_block_list *cache, + unsigned long index, + enum f2fs_cache_request_flag rflag) +{ + struct f2fs_cached_block *entry; + unsigned long flags; + bool access = rflag & F2FS_CACHE_ACCESS; + + spin_lock_irqsave(&cache->tree_lock, flags); + entry = radix_tree_lookup(&cache->root, index); + if (entry) { + f2fs_bug_on(cache->sbi, !f2fs_cache_refcount(entry)); + f2fs_cache_get(entry); + if (access && !f2fs_cache_test_referenced(entry)) + f2fs_cache_set_referenced(entry); + } else { + entry = ERR_PTR(-ENOENT); + } + spin_unlock_irqrestore(&cache->tree_lock, flags); + + return entry; +} + +struct f2fs_cached_block *f2fs_grab_cache( + struct f2fs_cached_block_list *cache, + unsigned long index, int flags) + +{ + struct f2fs_cached_block *entry, *new; + bool create = flags & F2FS_CACHE_CREATE; + bool nofail = flags & F2FS_CACHE_NOFAIL; + bool lock = flags & F2FS_CACHE_LOCK; + +repeat: + entry = f2fs_find_cache(cache, index, F2FS_CACHE_ACCESS); + if (!IS_ERR(entry)) + goto found; + + if (!create) + return ERR_PTR(-ENOENT); + + new = f2fs_create_cache(cache, index, nofail); + if (IS_ERR(new)) + return new; + + entry = f2fs_insert_cache(cache, index, new); +found: + if (lock) { + f2fs_lock_cache(entry); + /* has been truncated */ + if (entry->cache != cache) { + f2fs_put_cache(entry, true); + goto repeat; + } + } + return entry; +} + +bool f2fs_trylock_cache(struct f2fs_cached_block *entry) +{ + return !test_and_set_bit(F2FS_BLOCK_LOCKED, &entry->state); +} + +void f2fs_lock_cache(struct f2fs_cached_block *entry) +{ + wait_on_bit_lock(&entry->state, F2FS_BLOCK_LOCKED, + TASK_UNINTERRUPTIBLE); +} + +void f2fs_unlock_cache(struct f2fs_cached_block *entry) +{ + /* + * In asynchronous read I/O completion (e.g. from f2fs_ra_node_cache()), + * the I/O completion does not hold a reference of its own. Once + * clear_and_wake_up_bit() clears F2FS_BLOCK_LOCKED, a concurrent waiter + * in f2fs_lock_cache() (e.g. from f2fs_truncate_cache()) can wake up, + * truncate the entry, and drop the final reference before wake_up_bit() + * finishes. + * Pin the entry here to make sure it is not freed before wake_up_bit() + * completes. + */ + f2fs_cache_pin(entry); + clear_and_wake_up_bit(F2FS_BLOCK_LOCKED, &entry->state); + f2fs_cache_unpin(entry); +} + +bool f2fs_put_cache(struct f2fs_cached_block *entry, bool unlock) +{ + if (IS_ERR_OR_NULL(entry)) + return false; + if (unlock) + f2fs_unlock_cache(entry); + return f2fs_cache_put(entry); +} + +unsigned int f2fs_cache_gang_lookup(struct f2fs_cached_block_list *cache, + struct f2fs_cached_block **entries, + pgoff_t *index, unsigned long end) +{ + unsigned long flags; + unsigned int max_nr = min((unsigned long)F2FS_ONSTACK_CACHES, end - *index); + int nr, i; + + if (*index >= end || *index == ULONG_MAX) + return 0; + + spin_lock_irqsave(&cache->tree_lock, flags); + nr = radix_tree_gang_lookup(&cache->root, (void **)entries, + *index, max_nr); + if (!nr) + goto out_unlock; + + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; + + if (entry->index >= end) { + nr = i; + break; + } + f2fs_cache_get(entry); + } + if (nr) + *index = entries[nr - 1]->index + 1; +out_unlock: + spin_unlock_irqrestore(&cache->tree_lock, flags); + return nr; +} + +unsigned int f2fs_cache_gang_lookup_tag(struct f2fs_cached_block_list *cache, + struct f2fs_cached_block **entries, + pgoff_t *index, unsigned int max_nr, + int tag) +{ + unsigned long flags; + int nr, i; + + if (*index == ULONG_MAX) + return 0; + + spin_lock_irqsave(&cache->tree_lock, flags); + nr = radix_tree_gang_lookup_tag(&cache->root, (void **)entries, + *index, max_nr, tag); + if (!nr) + goto out; + + for (i = 0; i < nr; i++) + f2fs_cache_get(entries[i]); + *index = entries[nr - 1]->index + 1; +out: + spin_unlock_irqrestore(&cache->tree_lock, flags); + return nr; +} + +void f2fs_cache_gang_release(struct f2fs_cached_block **entries, + unsigned int nr_entries) +{ + int i; + + for (i = 0; i < nr_entries; i++) + f2fs_put_cache(entries[i], false); +} + +void f2fs_cache_wait_on_all_writeback(struct f2fs_cached_block_list *cache) +{ + unsigned long index = 0; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; + int nr, i; + +next: + nr = f2fs_cache_gang_lookup_tag(cache, entries, &index, + F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_WRITEBACK); + if (!nr) + return; + + for (i = 0; i < nr; i++) + f2fs_cache_wait_writeback(entries[i]); + f2fs_cache_gang_release(entries, nr); + + cond_resched(); + goto next; +} + +static void f2fs_do_truncate_cache(struct f2fs_cached_block *entry, + bool drop_dirty) +{ + struct f2fs_cached_block_list *cache = entry->cache; + unsigned long flags; + + if (drop_dirty) + goto drop_it; + + if (f2fs_cache_test_dirty(entry) || + f2fs_cache_test_writeback(entry)) + return; + +drop_it: + f2fs_cache_wait_writeback(entry); + f2fs_drop_cache_dirty(entry); + + spin_lock(&cache->list_lock); + spin_lock_irqsave(&cache->tree_lock, flags); + + f2fs_bug_on(cache->sbi, !entry->cache); + if (!radix_tree_delete(&cache->root, entry->index)) + f2fs_bug_on(cache->sbi, !entry->cache); + + entry->cache = NULL; + cache->num_entries--; + + atomic_dec(&entry->refcount); + f2fs_bug_on(cache->sbi, !f2fs_cache_refcount(entry)); + + f2fs_bug_on(cache->sbi, list_empty(&entry->list)); + list_del_init(&entry->list); + + spin_unlock_irqrestore(&cache->tree_lock, flags); + spin_unlock(&cache->list_lock); +} + +void f2fs_truncate_locked_cache(struct f2fs_cached_block *entry, + bool drop_dirty) +{ + if (!entry->cache) + return; + f2fs_do_truncate_cache(entry, drop_dirty); +} + +void f2fs_truncate_cache(struct f2fs_cached_block *entry, bool drop_dirty) +{ + f2fs_lock_cache(entry); + f2fs_truncate_locked_cache(entry, drop_dirty); + f2fs_unlock_cache(entry); +} + +static void f2fs_drop_cache(struct f2fs_cached_block_list *cache, + block_t blkaddr, bool drop_dirty) +{ + struct f2fs_cached_block *entry; + + entry = f2fs_find_cache(cache, blkaddr, 0); + if (IS_ERR(entry)) + return; + + f2fs_truncate_cache(entry, drop_dirty); + f2fs_put_cache(entry, false); +} + +void f2fs_drop_cache_range(struct f2fs_cached_block_list *cache, + unsigned long start, unsigned long len, bool drop_dirty) +{ + unsigned long index = start; + unsigned long end = (ULONG_MAX - start < len) ? + ULONG_MAX : (start + len); + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; + int nr, i; + + if (len == 1) + return f2fs_drop_cache(cache, index, drop_dirty); + +next: + nr = f2fs_cache_gang_lookup(cache, entries, &index, end); + if (!nr) + return; + + for (i = 0; i < nr; i++) + f2fs_truncate_cache(entries[i], drop_dirty); + f2fs_cache_gang_release(entries, nr); + + if (index < end) { + cond_resched(); + goto next; + } +} + +int f2fs_init_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block_list *cache, + enum f2fs_cache_type type) +{ + cache->sbi = sbi; + cache->type = type; + INIT_RADIX_TREE(&cache->root, GFP_ATOMIC); + spin_lock_init(&cache->tree_lock); + spin_lock_init(&cache->list_lock); + INIT_LIST_HEAD(&cache->lru_list); + cache->num_entries = 0; + + return 0; +} + +void f2fs_destroy_cache(struct f2fs_cached_block_list *cache) +{ + struct list_head *head = &cache->lru_list; + struct f2fs_cached_block *entry; + unsigned long flags; + + f2fs_cache_wait_on_all_writeback(cache); +next: + spin_lock(&cache->list_lock); + if (list_empty(head)) { + spin_unlock(&cache->list_lock); + return; + } + entry = list_first_entry(head, struct f2fs_cached_block, list); + + spin_lock_irqsave(&cache->tree_lock, flags); + radix_tree_delete(&cache->root, entry->index); + cache->num_entries--; + list_del_init(&entry->list); + spin_unlock_irqrestore(&cache->tree_lock, flags); + + spin_unlock(&cache->list_lock); + + /* wait on read cache IO */ + f2fs_lock_cache(entry); + /* wait on write cache IO */ + f2fs_cache_wait_writeback(entry); + f2fs_bug_on(cache->sbi, f2fs_cache_test_dirty(entry)); + f2fs_bug_on(cache->sbi, f2fs_cache_test_writeback(entry)); + f2fs_bug_on(cache->sbi, !list_empty(&entry->list)); + /* the I/O completion may not have unpinned the entry yet */ + f2fs_bug_on(cache->sbi, f2fs_cache_users(entry) != 1); + f2fs_put_cache(entry, true); + goto next; +} + +static unsigned long f2fs_do_shrink_cache(struct f2fs_cached_block_list *cache, + unsigned long nr_to_scan) +{ + struct f2fs_cached_block *entry, *next; + LIST_HEAD(dispose_list); + LIST_HEAD(keep_list); + unsigned long freed = 0; + unsigned long scanned = 0; + + /* Phase 1: Isolate candidate entries from LRU list into dispose_list */ + spin_lock(&cache->list_lock); + list_for_each_entry_safe(entry, next, &cache->lru_list, list) { + if (scanned >= cache->num_entries) + break; + if (scanned++ >= nr_to_scan) + break; + + /* If accessed, give it a second chance to rotate to tail */ + if (f2fs_cache_test_and_clear_referenced(entry)) { + list_move_tail(&entry->list, &cache->lru_list); + continue; + } + + if (f2fs_cache_test_dirty(entry) || + f2fs_cache_test_writeback(entry) || + f2fs_cache_test_locked(entry)) + continue; + + if (f2fs_cache_refcount(entry) != 1) + continue; + + list_move_tail(&entry->list, &dispose_list); + } + spin_unlock(&cache->list_lock); + + /* Phase 2: Process isolated candidates one by one */ + while (1) { + spin_lock(&cache->list_lock); + entry = list_first_entry_or_null(&dispose_list, + struct f2fs_cached_block, list); + if (!entry) { + spin_unlock(&cache->list_lock); + break; + } + f2fs_cache_get(entry); + list_move_tail(&entry->list, &keep_list); + spin_unlock(&cache->list_lock); + + if (!f2fs_trylock_cache(entry)) { + f2fs_put_cache(entry, false); + continue; + } + + /* the entry has been truncated */ + if (!entry->cache) { + f2fs_put_cache(entry, true); + continue; + } + /* + * at least there are shrinker, radix tree and another user + * has referenced the entry. + */ + if (f2fs_cache_refcount(entry) >= 3) { + f2fs_put_cache(entry, true); + continue; + } + + f2fs_do_truncate_cache(entry, false); + + if (f2fs_put_cache(entry, true)) + freed++; + } + + /* Phase 3: Splice un-reclaimed entries back onto cache->lru_list */ + if (!list_empty(&keep_list)) { + spin_lock(&cache->list_lock); + list_splice_tail(&keep_list, &cache->lru_list); + spin_unlock(&cache->list_lock); + } + + return freed; +} + +unsigned long f2fs_shrink_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan) +{ + unsigned long freed; + + freed = f2fs_do_shrink_cache(META_CACHE(sbi), nr_to_scan); + if (freed >= nr_to_scan) + return freed; + + freed += f2fs_do_shrink_cache(NODE_CACHE(sbi), nr_to_scan - freed); + if (freed >= nr_to_scan) + return freed; + + freed += f2fs_do_shrink_cache(COMPRESS_CACHE(sbi), nr_to_scan - freed); + return freed; +} + +unsigned long f2fs_shrink_meta_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan) +{ + return f2fs_do_shrink_cache(META_CACHE(sbi), nr_to_scan); +} + +unsigned long f2fs_shrink_node_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan) +{ + return f2fs_do_shrink_cache(NODE_CACHE(sbi), nr_to_scan); +} + +static inline unsigned long f2fs_total_cached_entries(struct f2fs_sb_info *sbi) +{ + unsigned long total = READ_ONCE(META_CACHE(sbi)->num_entries) + + READ_ONCE(NODE_CACHE(sbi)->num_entries); +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (test_opt(sbi, COMPRESS_CACHE)) + total += READ_ONCE(COMPRESS_CACHE(sbi)->num_entries); +#endif + return total; +} + +static inline s64 f2fs_nr_dirty_caches(struct f2fs_sb_info *sbi) +{ + return get_nr_caches(sbi, F2FS_DIRTY_META) + + get_nr_caches(sbi, F2FS_DIRTY_NODES); +} + +/* + * f2fs_write_{meta,node}_caches() skip writeback until there are enough + * dirty caches to gather, so don't wake up f2fs_writeback for nothing. + */ +static inline bool f2fs_cache_wb_collectable(struct f2fs_sb_info *sbi, + s64 nr_dirty, int type) +{ + return nr_dirty && nr_dirty >= nr_caches_to_skip(sbi, type); +} + +static bool f2fs_cache_wb_exceeded(struct f2fs_sb_info *sbi) +{ + struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread; + unsigned int total_threshold, dirty_threshold; + s64 nr_meta, nr_node; + + total_threshold = READ_ONCE(cache_thread->cache_wb_total_threshold); + dirty_threshold = READ_ONCE(cache_thread->cache_wb_dirty_threshold); + + /* + * Both thresholds are disabled, rely on periodic writeback only, + * unless checkpoint is disabled, as nothing else will flush dirty + * caches then. + */ + if (!total_threshold && !dirty_threshold && + !is_sbi_flag_set(sbi, SBI_CP_DISABLED)) + return false; + + if (total_threshold && + f2fs_total_cached_entries(sbi) < total_threshold) + return false; + + nr_meta = get_nr_caches(sbi, F2FS_DIRTY_META); + nr_node = get_nr_caches(sbi, F2FS_DIRTY_NODES); + + if (dirty_threshold && nr_meta + nr_node < dirty_threshold) + return false; + + return f2fs_cache_wb_collectable(sbi, nr_meta, META) || + f2fs_cache_wb_collectable(sbi, nr_node, NODE); +} + +static bool f2fs_should_wake_up_cache_wb(struct f2fs_sb_info *sbi) +{ + /* last round made no progress, wait for periodic writeback */ + if (READ_ONCE(sbi->cache_thread.cache_wb_throttled)) + return false; + + return f2fs_cache_wb_exceeded(sbi); +} + +static int f2fs_cache_writeback_kthread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread; + wait_queue_head_t *wq = &cache_thread->cache_wb_wq; + + set_freezable(); + + while (!kthread_should_stop()) { + unsigned int interval = READ_ONCE(cache_thread->cache_wb_interval); + long nr_meta = 0, nr_node = 0; + s64 nr_dirty; + + wait_event_freezable_timeout(*wq, + kthread_should_stop() || + f2fs_should_wake_up_cache_wb(sbi), + msecs_to_jiffies(interval)); + + if (kthread_should_stop()) + break; + + /* + * If thresholds are exceeded, throttle threshold based wakeup + * until a round makes progress, otherwise we may busy-loop if + * this round can not write anything, e.g. on cp_error, freezing, + * frozen fs, or lock contention. + */ + WRITE_ONCE(cache_thread->cache_wb_throttled, + f2fs_cache_wb_exceeded(sbi)); + + if (f2fs_readonly(sbi->sb)) + continue; + + if (f2fs_cp_error(sbi)) + continue; + + if (unlikely(freezing(current))) + continue; + + if (!sb_start_write_trylock(sbi->sb)) + continue; + + nr_dirty = f2fs_nr_dirty_caches(sbi); + + f2fs_write_meta_caches(sbi, &nr_meta); + f2fs_write_node_caches(sbi, &nr_node); + + sb_end_write(sbi->sb); + + /* + * Don't rely on the delta of dirty caches only, as caches may + * be dirtied concurrently; also count in caches cleaned by + * others, e.g. checkpoint triggered by f2fs_balance_fs_bg(). + */ + if (nr_meta || nr_node || f2fs_nr_dirty_caches(sbi) < nr_dirty) + WRITE_ONCE(cache_thread->cache_wb_throttled, false); + + cond_resched(); + } + return 0; +} + +void f2fs_wake_up_cache_wb(struct f2fs_sb_info *sbi) +{ + struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread; + + /* pairs with smp_store_release() in f2fs_start_cache_wb_thread() */ + if (!smp_load_acquire(&cache_thread->cache_wb_task)) + return; + + if (!f2fs_should_wake_up_cache_wb(sbi)) + return; + + if (wq_has_sleeper(&cache_thread->cache_wb_wq)) + wake_up(&cache_thread->cache_wb_wq); +} + +int f2fs_start_cache_wb_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread; + struct task_struct *task; + dev_t dev = sbi->sb->s_dev; + char name[36]; + + if (cache_thread->cache_wb_task) + return 0; + + init_waitqueue_head(&cache_thread->cache_wb_wq); + cache_thread->cache_wb_interval = DEF_DIRTY_CACHE_TIMEOUT; + cache_thread->cache_wb_dirty_threshold = DEF_CACHE_WB_DIRTY_THRESH; + cache_thread->cache_wb_total_threshold = 0; + cache_thread->cache_wb_throttled = false; + snprintf(name, sizeof(name), "f2fs_writeback-%u:%u", + MAJOR(dev), MINOR(dev)); + + task = kthread_run(f2fs_cache_writeback_kthread, sbi, "%s", name); + if (IS_ERR(task)) + return PTR_ERR(task); + + /* pairs with smp_load_acquire() in f2fs_wake_up_cache_wb() */ + smp_store_release(&cache_thread->cache_wb_task, task); + return 0; +} + +void f2fs_stop_cache_wb_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread; + + if (!cache_thread->cache_wb_task) + return; + + kthread_stop(cache_thread->cache_wb_task); + WRITE_ONCE(cache_thread->cache_wb_task, NULL); +} diff --git a/fs/f2fs/cache.h b/fs/f2fs/cache.h new file mode 100644 index 000000000000..5408727dd321 --- /dev/null +++ b/fs/f2fs/cache.h @@ -0,0 +1,252 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Google LLC + * Author: Chao Yu + */ +#ifndef _LINUX_F2FS_CACHE_H +#define _LINUX_F2FS_CACHE_H + +#include +#include +#include +#include +#include +#include +#include + +struct f2fs_rwsem; +struct f2fs_io_info; +enum page_type; + +/* Represents a single cached block (meta, node or compress) */ +struct f2fs_cached_block { + struct list_head list; /* LRU list head */ + struct f2fs_cached_block_list *cache; /* parent cache list */ + union { + struct f2fs_cached_block *next_entry;/* chain for merged BIO */ + nid_t ino; /* inode number for compress cache */ + }; + unsigned long index; /* key in radix tree, (meta/compress: pba, node: nid) */ + unsigned long state; /* cache entry state (e.g., Dirty, UpToDate) */ + void *data; /* blocksize-aligned memory (4KB or 16KB) */ + atomic_t refcount; /* reference count */ +}; + +struct f2fs_sb_info; + +enum f2fs_cache_type { + F2FS_META_CACHE, + F2FS_NODE_CACHE, + F2FS_COMPRESS_CACHE, + NR_CACHE_TYPES, +}; + +/* Main cache control structure (per sb_info) */ +struct f2fs_cached_block_list { + struct f2fs_sb_info *sbi; /* Pointer to f2fs_sb_info */ + struct radix_tree_root root; /* Radix tree for cache lookup */ + spinlock_t tree_lock; /* Lock for radix tree */ + struct list_head lru_list; /* Single global LRU list */ + spinlock_t list_lock; /* Lock for LRU list */ + enum f2fs_cache_type type; /* Cache type (Node, Meta, Compress) */ + unsigned long num_entries; /* Current number of entries */ +}; + +#define IS_META_CACHE(cache) ((cache)->type == F2FS_META_CACHE) +#define IS_NODE_CACHE(cache) ((cache)->type == F2FS_NODE_CACHE) +#define IS_COMPRESS_CACHE(cache) ((cache)->type == F2FS_COMPRESS_CACHE) + +/* Flags for f2fs_cached_block state */ +enum f2fs_cached_state { + F2FS_BLOCK_LOCKED, /* cache entry is locked */ + F2FS_BLOCK_UPTODATE, /* cache data is valid */ + F2FS_BLOCK_DIRTY, /* cache data is dirty, need to writeback the data */ + F2FS_BLOCK_WRITEBACK, /* cache data is writeback state */ + F2FS_BLOCK_REFERENCED, /* cache was accessed recently, shrinker will skip it for once */ + F2FS_BLOCK_INLINE_DATA, /* indicate inline data */ +}; + +enum { + __F2FS_CACHE_CREATE, /* create the cache if there is no cache entry */ + __F2FS_CACHE_LOCK, /* get and lock the cache entry */ + __F2FS_CACHE_NOFAIL, /* do not allow failure */ + __F2FS_CACHE_ACCESS, /* give a chance to add referenced tag */ +}; + +enum f2fs_cache_request_flag { + F2FS_CACHE_CREATE = 1 << __F2FS_CACHE_CREATE, + F2FS_CACHE_LOCK = 1 << __F2FS_CACHE_LOCK, + F2FS_CACHE_NOFAIL = 1 << __F2FS_CACHE_NOFAIL, + F2FS_CACHE_ACCESS = 1 << __F2FS_CACHE_ACCESS, +}; + +#define F2FS_CACHE_LOCK_CREATE (F2FS_CACHE_LOCK | F2FS_CACHE_CREATE) + +#define F2FS_ONSTACK_CACHES (32) + +#define METADATA_CACHE_SHRINK_NUMBER (128) + +#define F2FS_CACHE_FLAG_TEST_FUNC(name, flagname) \ +static inline bool f2fs_cache_test_##name( \ + const struct f2fs_cached_block *entry) \ +{ \ + return test_bit(F2FS_BLOCK_##flagname, &entry->state); \ +} \ + +#define F2FS_CACHE_FLAG_SET_FUNC(name, flagname) \ +static inline void f2fs_cache_set_##name( \ + struct f2fs_cached_block *entry) \ +{ \ + set_bit(F2FS_BLOCK_##flagname, &entry->state); \ +} \ + +#define F2FS_CACHE_FLAG_CLEAR_FUNC(name, flagname) \ +static inline void f2fs_cache_clear_##name( \ + struct f2fs_cached_block *entry) \ +{ \ + clear_bit(F2FS_BLOCK_##flagname, &entry->state); \ +} \ + +#define F2FS_CACHE_FLAG_TEST_AND_SET_FUNC(name, flagname) \ +static inline bool f2fs_cache_test_and_set_##name( \ + struct f2fs_cached_block *entry) \ +{ \ + return test_and_set_bit(F2FS_BLOCK_##flagname, &entry->state); \ +} \ + +#define F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(name, flagname) \ +static inline bool f2fs_cache_test_and_clear_##name( \ + struct f2fs_cached_block *entry) \ +{ \ + return test_and_clear_bit(F2FS_BLOCK_##flagname, &entry->state);\ +} \ + +F2FS_CACHE_FLAG_TEST_FUNC(locked, LOCKED); +F2FS_CACHE_FLAG_SET_FUNC(locked, LOCKED); +F2FS_CACHE_FLAG_CLEAR_FUNC(locked, LOCKED); + +F2FS_CACHE_FLAG_TEST_FUNC(uptodate, UPTODATE); +F2FS_CACHE_FLAG_SET_FUNC(uptodate, UPTODATE); +F2FS_CACHE_FLAG_CLEAR_FUNC(uptodate, UPTODATE); + +F2FS_CACHE_FLAG_TEST_FUNC(dirty, DIRTY); +F2FS_CACHE_FLAG_SET_FUNC(dirty, DIRTY); +F2FS_CACHE_FLAG_CLEAR_FUNC(dirty, DIRTY); +F2FS_CACHE_FLAG_TEST_AND_SET_FUNC(dirty, DIRTY); +F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(dirty, DIRTY); + +F2FS_CACHE_FLAG_TEST_FUNC(writeback, WRITEBACK); +F2FS_CACHE_FLAG_SET_FUNC(writeback, WRITEBACK); +F2FS_CACHE_FLAG_CLEAR_FUNC(writeback, WRITEBACK); + +F2FS_CACHE_FLAG_TEST_FUNC(inline, INLINE_DATA); +F2FS_CACHE_FLAG_SET_FUNC(inline, INLINE_DATA); +F2FS_CACHE_FLAG_CLEAR_FUNC(inline, INLINE_DATA); + +F2FS_CACHE_FLAG_TEST_FUNC(referenced, REFERENCED); +F2FS_CACHE_FLAG_SET_FUNC(referenced, REFERENCED); +F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(referenced, REFERENCED); + +static inline void *cache_address(const struct f2fs_cached_block *entry) +{ + return entry->data; +} + +#define CACHED_NODE(entry) ((struct f2fs_node *)(cache_address(entry))) + +static inline struct folio *cache_folio(const struct f2fs_cached_block *entry) +{ + return virt_to_folio(entry->data); +} + +int f2fs_init_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block_list *cache, + enum f2fs_cache_type type); +void f2fs_destroy_cache(struct f2fs_cached_block_list *cache); +void f2fs_cache_get(struct f2fs_cached_block *entry); +struct f2fs_cached_block *f2fs_find_cache( + struct f2fs_cached_block_list *cache, + unsigned long index, + enum f2fs_cache_request_flag rflag); +#define F2FS_CACHE_TAG_NONE 0 +#define F2FS_CACHE_TAG_DIRTY 1 +#define F2FS_CACHE_TAG_WRITEBACK 2 + +bool f2fs_trylock_cache(struct f2fs_cached_block *entry); +void f2fs_lock_cache(struct f2fs_cached_block *entry); +void f2fs_unlock_cache(struct f2fs_cached_block *entry); +bool f2fs_put_cache(struct f2fs_cached_block *entry, bool unlock); +bool f2fs_mark_cache_dirty(struct f2fs_cached_block *entry); +void f2fs_drop_cache_dirty(struct f2fs_cached_block *entry); +void f2fs_start_cache_writeback(struct f2fs_cached_block *entry); +void f2fs_end_cache_writeback(struct f2fs_cached_block *entry); +unsigned int f2fs_cache_gang_lookup(struct f2fs_cached_block_list *cache, + struct f2fs_cached_block **entries, + pgoff_t *index, unsigned long end); +unsigned int f2fs_cache_gang_lookup_tag(struct f2fs_cached_block_list *cache, + struct f2fs_cached_block **results, pgoff_t *first_index, + unsigned int max_items, int tag); +void f2fs_cache_gang_release(struct f2fs_cached_block **entries, + unsigned int nr_entries); +void f2fs_cache_wait_on_all_writeback(struct f2fs_cached_block_list *cache); +void f2fs_cache_wait_writeback_cond(struct f2fs_cached_block *entry, + enum page_type type); +void f2fs_cache_wait_writeback(struct f2fs_cached_block *entry); +void f2fs_cache_update_tag(struct f2fs_cached_block *entry, + unsigned int clear_from, unsigned int set_to); +struct f2fs_cached_block *f2fs_grab_cache(struct f2fs_cached_block_list *cache, + unsigned long index, int flags); +void f2fs_truncate_locked_cache(struct f2fs_cached_block *entry, + bool drop_dirty); +void f2fs_truncate_cache(struct f2fs_cached_block *entry, bool drop_dirty); +void f2fs_drop_cache_range(struct f2fs_cached_block_list *cache, + unsigned long start, unsigned long len, bool drop_dirty); + +#define META_CACHE(sbi) (&(sbi)->meta_blocks) +#define NODE_CACHE(sbi) (&(sbi)->node_blocks) +#define COMPRESS_CACHE(sbi) (&(sbi)->compress_blocks) + +#define f2fs_find_meta_cache(sbi, blkaddr) \ + f2fs_find_cache(META_CACHE(sbi), blkaddr, 0) +#define f2fs_invalidate_meta_caches(sbi, start, len) \ + f2fs_drop_cache_range(META_CACHE(sbi), start, len, false) +#define f2fs_truncate_meta_caches(sbi, start, len) \ + f2fs_drop_cache_range(META_CACHE(sbi), start, len, true) + +#define f2fs_grab_node_cache(sbi, blkaddr) \ + f2fs_grab_cache(NODE_CACHE(sbi), blkaddr, \ + F2FS_CACHE_LOCK_CREATE) +#define f2fs_find_node_cache(sbi, blkaddr) \ + f2fs_find_cache(NODE_CACHE(sbi), blkaddr, F2FS_CACHE_ACCESS) +#define f2fs_invalidate_node_cache(sbi, blkaddr) \ + f2fs_drop_cache_range(NODE_CACHE(sbi), blkaddr, 1, false) +#define f2fs_truncate_node_caches(sbi, start, len) \ + f2fs_drop_cache_range(NODE_CACHE(sbi), start, len, true) + +unsigned long f2fs_shrink_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan); +unsigned long f2fs_shrink_meta_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan); +unsigned long f2fs_shrink_node_cache(struct f2fs_sb_info *sbi, + unsigned long nr_to_scan); + +#define DEF_DIRTY_CACHE_TIMEOUT 5000 +#define MIN_DIRTY_CACHE_TIMEOUT 100 +#define MAX_DIRTY_CACHE_TIMEOUT 30000 + +#define DEF_CACHE_WB_DIRTY_THRESH (8192) + +struct f2fs_cache_kthread { + struct task_struct *cache_wb_task; + wait_queue_head_t cache_wb_wq; + unsigned int cache_wb_interval; + unsigned int cache_wb_dirty_threshold; + unsigned int cache_wb_total_threshold; + bool cache_wb_throttled; /* last round made no progress */ +}; + +int f2fs_start_cache_wb_thread(struct f2fs_sb_info *sbi); +void f2fs_stop_cache_wb_thread(struct f2fs_sb_info *sbi); +void f2fs_wake_up_cache_wb(struct f2fs_sb_info *sbi); + +#endif /* _LINUX_F2FS_CACHE_H */ diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 4b59f30ef45d..5c5b45d3c67b 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -7,7 +7,6 @@ */ #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include #include "f2fs.h" #include "node.h" @@ -234,29 +234,29 @@ static struct kmem_cache *ino_entry_slab; struct kmem_cache *f2fs_inode_entry_slab; /* - * We guarantee no failure on the returned page. + * We guarantee no failure on the returned cache. */ -struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index) +struct f2fs_cached_block *f2fs_grab_meta_cache(struct f2fs_sb_info *sbi, + pgoff_t index) { - struct address_space *mapping = META_MAPPING(sbi); - struct folio *folio; + struct f2fs_cached_block *entry; repeat: - folio = f2fs_grab_cache_folio(mapping, index, false); - if (IS_ERR(folio)) { + entry = f2fs_grab_cache(META_CACHE(sbi), index, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(entry)) { cond_resched(); goto repeat; } - f2fs_folio_wait_writeback(folio, META, true, true); - if (!folio_test_uptodate(folio)) - folio_mark_uptodate(folio); - return folio; + f2fs_cache_wait_writeback(entry); + if (!f2fs_cache_test_uptodate(entry)) + f2fs_cache_set_uptodate(entry); + return entry; } -static struct folio *__get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index, - bool is_meta) +static struct f2fs_cached_block *__get_meta_cache(struct f2fs_sb_info *sbi, + pgoff_t index, bool is_meta) { - struct address_space *mapping = META_MAPPING(sbi); - struct folio *folio; + struct f2fs_cached_block *entry; struct f2fs_io_info fio = { .sbi = sbi, .type = META, @@ -266,70 +266,75 @@ static struct folio *__get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index, .new_blkaddr = index, .encrypted_page = NULL, .is_por = !is_meta ? 1 : 0, + .is_cache = 1, }; int err; if (unlikely(!is_meta)) fio.op_flags &= ~REQ_META; repeat: - folio = f2fs_grab_cache_folio(mapping, index, false); - if (IS_ERR(folio)) { + entry = f2fs_grab_cache(META_CACHE(sbi), index, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(entry)) { cond_resched(); goto repeat; } - if (folio_test_uptodate(folio)) + if (f2fs_cache_test_uptodate(entry)) goto out; - fio.folio = folio; + fio.cache_entry = entry; - err = f2fs_submit_page_bio(&fio); + err = f2fs_submit_cache_read(&fio); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return ERR_PTR(err); } - f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE(sbi)); - folio_lock(folio); - if (unlikely(!is_meta_folio(folio))) { - f2fs_folio_put(folio, true); + f2fs_lock_cache(entry); + if (unlikely(!f2fs_is_meta_cache(entry))) { + f2fs_put_cache(entry, true); goto repeat; } - if (unlikely(!folio_test_uptodate(folio))) { - f2fs_handle_page_eio(sbi, folio, META); - f2fs_folio_put(folio, true); + if (unlikely(!f2fs_cache_test_uptodate(entry))) { + f2fs_handle_page_eio(sbi, entry->index, META); + f2fs_put_cache(entry, true); return ERR_PTR(-EIO); } out: - return folio; + return entry; } -struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index) +struct f2fs_cached_block *f2fs_get_meta_cache(struct f2fs_sb_info *sbi, + pgoff_t index) { - return __get_meta_folio(sbi, index, true); + return __get_meta_cache(sbi, index, true); } -struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index) +struct f2fs_cached_block *f2fs_get_meta_cache_retry(struct f2fs_sb_info *sbi, + pgoff_t index) { - struct folio *folio; + struct f2fs_cached_block *entry; int count = 0; retry: - folio = __get_meta_folio(sbi, index, true); - if (IS_ERR(folio)) { - if (PTR_ERR(folio) == -EIO && + entry = __get_meta_cache(sbi, index, true); + if (IS_ERR(entry)) { + if (PTR_ERR(entry) == -EIO && ++count <= DEFAULT_RETRY_IO_COUNT) goto retry; f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE); } - return folio; + return entry; } /* for POR only */ -struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index) +struct f2fs_cached_block *f2fs_get_tmp_cache(struct f2fs_sb_info *sbi, + pgoff_t index) { - return __get_meta_folio(sbi, index, false); + return __get_meta_cache(sbi, index, false); } static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr, @@ -445,11 +450,12 @@ bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi, } /* - * Readahead CP/NAT/SIT/SSA/POR pages + * Readahead CP/NAT/SIT/SSA/POR blocks */ -int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, - int type, bool sync) +int f2fs_ra_meta_caches(struct f2fs_sb_info *sbi, block_t start, + int nrblocks, int type, bool sync) { + struct f2fs_cached_block_list *cache = META_CACHE(sbi); block_t blkno = start; struct f2fs_io_info fio = { .sbi = sbi, @@ -459,6 +465,7 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, .encrypted_page = NULL, .in_list = 0, .is_por = (type == META_POR) ? 1 : 0, + .is_cache = 1, }; struct blk_plug plug; int err; @@ -467,8 +474,8 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, fio.op_flags &= ~REQ_META; blk_start_plug(&plug); - for (; nrpages-- > 0; blkno++) { - struct folio *folio; + for (; nrblocks-- > 0; blkno++) { + struct f2fs_cached_block *entry; if (!f2fs_is_valid_blkaddr(sbi, blkno, type)) goto out; @@ -476,18 +483,18 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, switch (type) { case META_NAT: if (unlikely(blkno >= - NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid))) + NAT_BLOCK_OFFSET(sbi, NM_I(sbi)->max_nid))) blkno = 0; /* get nat block addr */ fio.new_blkaddr = current_nat_addr(sbi, - blkno * NAT_ENTRY_PER_BLOCK); + blkno * NAT_ENTRY_PER_BLOCK(sbi)); break; case META_SIT: if (unlikely(blkno >= TOTAL_SEGS(sbi))) goto out; /* get sit block addr */ fio.new_blkaddr = current_sit_addr(sbi, - blkno * SIT_ENTRY_PER_BLOCK); + blkno * SIT_ENTRY_PER_BLOCK(sbi)); break; case META_SSA: case META_CP: @@ -495,62 +502,63 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, fio.new_blkaddr = blkno; break; default: - BUG(); + f2fs_bug_on(sbi, 1); } - folio = f2fs_grab_cache_folio(META_MAPPING(sbi), - fio.new_blkaddr, false); - if (IS_ERR(folio)) + entry = f2fs_grab_cache(cache, fio.new_blkaddr, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(entry)) continue; - if (folio_test_uptodate(folio)) { - f2fs_folio_put(folio, true); + if (f2fs_cache_test_uptodate(entry)) { + f2fs_put_cache(entry, true); continue; } - fio.folio = folio; - err = f2fs_submit_page_bio(&fio); - f2fs_folio_put(folio, err ? true : false); + fio.cache_entry = entry; + err = f2fs_submit_cache_read(&fio); + f2fs_put_cache(entry, err ? true : false); if (!err) f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, - F2FS_BLKSIZE); + F2FS_BLKSIZE(sbi)); } out: blk_finish_plug(&plug); return blkno - start; } -void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index, - unsigned int ra_blocks) +void f2fs_ra_meta_caches_cond(struct f2fs_sb_info *sbi, pgoff_t index, + unsigned int ra_blocks) { - struct folio *folio; + struct f2fs_cached_block *entry; bool readahead = false; if (ra_blocks == RECOVERY_MIN_RA_BLOCKS) return; - folio = filemap_get_folio(META_MAPPING(sbi), index); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) + entry = f2fs_find_cache(META_CACHE(sbi), index, 0); + if (IS_ERR(entry) || !f2fs_cache_test_uptodate(entry)) readahead = true; - f2fs_folio_put(folio, false); + f2fs_put_cache(entry, false); if (readahead) - f2fs_ra_meta_pages(sbi, index, ra_blocks, META_POR, true); + f2fs_ra_meta_caches(sbi, index, ra_blocks, META_POR, true); } -static bool __f2fs_write_meta_folio(struct folio *folio, - struct writeback_control *wbc, +static bool __f2fs_write_meta_cache(struct f2fs_cached_block *entry, enum iostat_type io_type) { - struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct f2fs_sb_info *sbi = entry->cache->sbi; - trace_f2fs_writepage(folio, META); + trace_f2fs_write_cache(entry, META); if (unlikely(f2fs_cp_error(sbi))) { if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) { - folio_clear_uptodate(folio); - dec_page_count(sbi, F2FS_DIRTY_META); - folio_unlock(folio); + f2fs_cache_clear_uptodate(entry); + dec_cache_count(sbi, F2FS_DIRTY_META); + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, + F2FS_CACHE_TAG_NONE); + f2fs_unlock_cache(entry); return true; } goto redirty_out; @@ -558,10 +566,10 @@ static bool __f2fs_write_meta_folio(struct folio *folio, if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) goto redirty_out; - f2fs_do_write_meta_page(sbi, folio, io_type); - dec_page_count(sbi, F2FS_DIRTY_META); + f2fs_do_write_meta_cache(sbi, entry, io_type); + dec_cache_count(sbi, F2FS_DIRTY_META); - folio_unlock(folio); + f2fs_unlock_cache(entry); if (unlikely(f2fs_cp_error(sbi))) f2fs_submit_merged_write(sbi, META); @@ -569,102 +577,100 @@ static bool __f2fs_write_meta_folio(struct folio *folio, return true; redirty_out: - folio_redirty_for_writepage(wbc, folio); + f2fs_cache_set_dirty(entry); return false; } -static int f2fs_write_meta_pages(struct address_space *mapping, - struct writeback_control *wbc) +void f2fs_write_meta_caches(struct f2fs_sb_info *sbi, long *nr_written) { - struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); struct f2fs_lock_context lc; - long diff, written; + long nr_to_write = LONG_MAX; + long nwritten; if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) - goto skip_write; + return; - /* collect a number of dirty meta pages and write together */ - if (wbc->sync_mode != WB_SYNC_ALL && - get_pages(sbi, F2FS_DIRTY_META) < - nr_pages_to_skip(sbi, META)) - goto skip_write; + /* collect a number of dirty meta caches and write together */ + if (get_nr_caches(sbi, F2FS_DIRTY_META) < + nr_caches_to_skip(sbi, META)) + return; - /* if locked failed, cp will flush dirty pages instead */ + /* if locked failed, cp will flush dirty caches instead */ if (!f2fs_down_write_trylock_trace(&sbi->cp_global_sem, &lc)) - goto skip_write; + return; - trace_f2fs_writepages(mapping->host, wbc, META); - diff = nr_pages_to_write(sbi, META, wbc); - written = f2fs_sync_meta_pages(sbi, wbc->nr_to_write, FS_META_IO); + nr_to_write = adjust_flush_cache_number(sbi, META); + nwritten = f2fs_sync_meta_caches(sbi, nr_to_write, false, FS_META_IO); f2fs_up_write_trace(&sbi->cp_global_sem, &lc); - wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff); - return 0; -skip_write: - wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META); - trace_f2fs_writepages(mapping->host, wbc, META); - return 0; + if (nr_written) + *nr_written = nwritten; } -long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write, - enum iostat_type io_type) +long f2fs_sync_meta_caches(struct f2fs_sb_info *sbi, long nr_to_write, + bool sync, enum iostat_type io_type) { - struct address_space *mapping = META_MAPPING(sbi); - pgoff_t index = 0, prev = ULONG_MAX; - struct folio_batch fbatch; + pgoff_t index = 0, next = ULONG_MAX; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; long nwritten = 0; - int nr_folios; - struct writeback_control wbc = {}; + int nr; struct blk_plug plug; + bool background = nr_to_write != LONG_MAX; - folio_batch_init(&fbatch); + trace_f2fs_write_caches(sbi, nr_to_write, 0, META); blk_start_plug(&plug); - while ((nr_folios = filemap_get_folios_tag(mapping, &index, - (pgoff_t)-1, - PAGECACHE_TAG_DIRTY, &fbatch))) { + while ((nr = f2fs_cache_gang_lookup_tag(META_CACHE(sbi), entries, + &index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) { int i; - for (i = 0; i < nr_folios; i++) { - struct folio *folio = fbatch.folios[i]; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; - if (nr_to_write != LONG_MAX && i != 0 && - folio->index != prev + - folio_nr_pages(fbatch.folios[i-1])) { - folio_batch_release(&fbatch); + if (background && unlikely(freezing(current))) { + f2fs_cache_gang_release(entries, nr); goto stop; } - folio_lock(folio); + if (background && next != ULONG_MAX && + entry->index != next) { + f2fs_cache_gang_release(entries, nr); + goto stop; + } - if (unlikely(!is_meta_folio(folio))) { + f2fs_lock_cache(entry); + + if (unlikely(!f2fs_is_meta_cache(entry))) { continue_unlock: - folio_unlock(folio); + f2fs_unlock_cache(entry); continue; } - if (!folio_test_dirty(folio)) { + if (!f2fs_cache_test_dirty(entry)) { /* someone wrote it for us */ goto continue_unlock; } - f2fs_folio_wait_writeback(folio, META, true, true); + f2fs_cache_wait_writeback(entry); - if (!folio_clear_dirty_for_io(folio)) + if (!f2fs_cache_test_and_clear_dirty(entry)) goto continue_unlock; - if (!__f2fs_write_meta_folio(folio, &wbc, - io_type)) { - folio_unlock(folio); - break; + if (!__f2fs_write_meta_cache(entry, io_type)) { + f2fs_unlock_cache(entry); + f2fs_cache_gang_release(entries, nr); + goto stop; } - nwritten += folio_nr_pages(folio); - prev = folio->index; + nwritten++; + next = entry->index + 1; if (unlikely(nwritten >= nr_to_write)) break; } - folio_batch_release(&fbatch); + f2fs_cache_gang_release(entries, nr); cond_resched(); + + if (unlikely(nwritten >= nr_to_write)) + break; } stop: if (nwritten) @@ -672,32 +678,11 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write, blk_finish_plug(&plug); + trace_f2fs_write_caches(sbi, nr_to_write, nwritten, META); + return nwritten; } -static bool f2fs_dirty_meta_folio(struct address_space *mapping, - struct folio *folio) -{ - trace_f2fs_set_page_dirty(folio, META); - - if (!folio_test_uptodate(folio)) - folio_mark_uptodate(folio); - if (filemap_dirty_folio(mapping, folio)) { - inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META); - folio_set_f2fs_reference(folio); - return true; - } - return false; -} - -const struct address_space_operations f2fs_meta_aops = { - .writepages = f2fs_write_meta_pages, - .dirty_folio = f2fs_dirty_meta_folio, - .invalidate_folio = f2fs_invalidate_folio, - .release_folio = f2fs_release_folio, - .migrate_folio = filemap_migrate_folio, -}; - static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, unsigned int devidx, int type) { @@ -825,15 +810,6 @@ static void __clear_ino_bitmap(struct f2fs_sb_info *sbi, nid_t ino, int type) spin_unlock(&im->ino_lock); } -static void f2fs_wait_for_inode_record(struct f2fs_sb_info *sbi, int mode) -{ - if (mode != APPEND_INO && mode != UPDATE_INO) - return; - - /* Let's wait for some pending updates for APPEND_INO and UPDATE_INO. */ - flush_workqueue(sbi->evict_wq); -} - static void __f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, unsigned int devidx, int type) { @@ -887,8 +863,6 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) for (i = all ? ORPHAN_INO : FLUSH_INO; i <= FLUSH_INO; i++) { struct inode_management *im = &sbi->im[i]; - f2fs_wait_for_inode_record(sbi, i); - spin_lock(&im->ino_lock); list_for_each_entry_safe(e, tmp, &im->ino_list, list) { list_del(&e->list); @@ -899,6 +873,9 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) spin_unlock(&im->ino_lock); } + /* Wait for pending APPEND/UPDATE inode state updates. */ + flush_workqueue(sbi->evict_wq); + for (i = APPEND_INO; i < MAX_INO_ENTRY; i++) { struct inode_management *im = &sbi->im[i]; @@ -964,7 +941,7 @@ void f2fs_add_orphan_inode(struct inode *inode) { /* add new orphan ino entry into list */ f2fs_add_ino_entry(F2FS_I_SB(inode), inode->i_ino, ORPHAN_INO); - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); } void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) @@ -1037,28 +1014,30 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi); orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi); - f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true); + f2fs_ra_meta_caches(sbi, start_blk, orphan_blocks, META_CP, true); for (i = 0; i < orphan_blocks; i++) { - struct folio *folio; + struct f2fs_cached_block *entry; struct f2fs_orphan_block *orphan_blk; + struct f2fs_orphan_footer *footer; unsigned int entry_count; - folio = f2fs_get_meta_folio(sbi, start_blk + i); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + entry = f2fs_get_meta_cache(sbi, start_blk + i); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); goto out; } - orphan_blk = folio_address(folio); - entry_count = le32_to_cpu(orphan_blk->entry_count); - if (entry_count > F2FS_ORPHANS_PER_BLOCK) { + orphan_blk = cache_address(entry); + footer = f2fs_orphan_footer(orphan_blk, sbi); + entry_count = le32_to_cpu(footer->entry_count); + if (entry_count > F2FS_ORPHANS_PER_BLOCK(sbi)) { f2fs_err(sbi, "invalid orphan inode entry count %u", entry_count); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INCONSISTENT_ORPHAN); err = -EFSCORRUPTED; - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); goto out; } @@ -1067,11 +1046,11 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) err = recover_orphan_inode(sbi, ino); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); goto out; } } - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } /* clear Orphan Flag */ clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG); @@ -1085,96 +1064,99 @@ static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk) { struct list_head *head; struct f2fs_orphan_block *orphan_blk = NULL; + struct f2fs_orphan_footer *footer = NULL; unsigned int nentries = 0; unsigned short index = 1; unsigned short orphan_blocks; - struct folio *folio = NULL; struct ino_entry *orphan = NULL; struct inode_management *im = &sbi->im[ORPHAN_INO]; + struct f2fs_cached_block *entry = NULL; - orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num); + orphan_blocks = GET_ORPHAN_BLOCKS(sbi, im->ino_num); /* * we don't need to do spin_lock(&im->ino_lock) here, since all the * orphan inode operations are covered under f2fs_lock_op(). - * And, spin_lock should be avoided due to page operations below. + * And, spin_lock should be avoided due to cache operations below. */ head = &im->ino_list; /* loop for each orphan inode entry and write them in journal block */ list_for_each_entry(orphan, head, list) { - if (!folio) { - folio = f2fs_grab_meta_folio(sbi, start_blk++); - orphan_blk = folio_address(folio); - memset(orphan_blk, 0, sizeof(*orphan_blk)); + if (!entry) { + entry = f2fs_grab_meta_cache(sbi, start_blk++); + orphan_blk = cache_address(entry); + footer = f2fs_orphan_footer(orphan_blk, sbi); + memset(orphan_blk, 0, sbi->blocksize); } orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino); - if (nentries == F2FS_ORPHANS_PER_BLOCK) { + if (nentries == F2FS_ORPHANS_PER_BLOCK(sbi)) { /* - * an orphan block is full of 1020 entries, + * an orphan block is full, * then we need to flush current orphan blocks * and bring another one in memory */ - orphan_blk->blk_addr = cpu_to_le16(index); - orphan_blk->blk_count = cpu_to_le16(orphan_blocks); - orphan_blk->entry_count = cpu_to_le32(nentries); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + footer->blk_addr = cpu_to_le16(index); + footer->blk_count = cpu_to_le16(orphan_blocks); + footer->entry_count = cpu_to_le32(nentries); + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); index++; nentries = 0; - folio = NULL; + entry = NULL; } } - if (folio) { - orphan_blk->blk_addr = cpu_to_le16(index); - orphan_blk->blk_count = cpu_to_le16(orphan_blocks); - orphan_blk->entry_count = cpu_to_le32(nentries); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + if (entry) { + footer->blk_addr = cpu_to_le16(index); + footer->blk_count = cpu_to_le16(orphan_blocks); + footer->entry_count = cpu_to_le32(nentries); + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); } } -static __u32 f2fs_checkpoint_chksum(struct f2fs_checkpoint *ckpt) +static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi, + struct f2fs_checkpoint *ckpt) { unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset); __u32 chksum; chksum = f2fs_crc32(ckpt, chksum_ofs); - if (chksum_ofs < CP_CHKSUM_OFFSET) { + if (chksum_ofs < CP_CHKSUM_OFFSET(sbi)) { chksum_ofs += sizeof(chksum); chksum = f2fs_chksum(chksum, (__u8 *)ckpt + chksum_ofs, - F2FS_BLKSIZE - chksum_ofs); + F2FS_BLKSIZE(sbi) - chksum_ofs); } return chksum; } static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr, - struct f2fs_checkpoint **cp_block, struct folio **cp_folio, + struct f2fs_checkpoint **cp_block, struct f2fs_cached_block **cp_entry, unsigned long long *version) { size_t crc_offset = 0; __u32 crc; - *cp_folio = f2fs_get_meta_folio(sbi, cp_addr); - if (IS_ERR(*cp_folio)) - return PTR_ERR(*cp_folio); + *cp_entry = f2fs_get_meta_cache(sbi, cp_addr); + if (IS_ERR(*cp_entry)) + return PTR_ERR(*cp_entry); - *cp_block = folio_address(*cp_folio); + *cp_block = cache_address(*cp_entry); crc_offset = le32_to_cpu((*cp_block)->checksum_offset); if (crc_offset < CP_MIN_CHKSUM_OFFSET || - crc_offset > CP_CHKSUM_OFFSET) { - f2fs_folio_put(*cp_folio, true); + crc_offset > CP_CHKSUM_OFFSET(sbi)) { + f2fs_put_cache(*cp_entry, true); f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset); return -EINVAL; } - crc = f2fs_checkpoint_chksum(*cp_block); + crc = f2fs_checkpoint_chksum(sbi, *cp_block); if (crc != cur_cp_crc(*cp_block)) { - f2fs_folio_put(*cp_folio, true); + f2fs_put_cache(*cp_entry, true); f2fs_warn(sbi, "invalid crc value"); return -EINVAL; } @@ -1183,17 +1165,17 @@ static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr, return 0; } -static struct folio *validate_checkpoint(struct f2fs_sb_info *sbi, +static struct f2fs_cached_block *validate_checkpoint(struct f2fs_sb_info *sbi, block_t cp_addr, unsigned long long *version) { - struct folio *cp_folio_1 = NULL, *cp_folio_2 = NULL; + struct f2fs_cached_block *cp_entry_1 = NULL, *cp_entry_2 = NULL; struct f2fs_checkpoint *cp_block = NULL; unsigned long long cur_version = 0, pre_version = 0; unsigned int cp_blocks; int err; err = get_checkpoint_version(sbi, cp_addr, &cp_block, - &cp_folio_1, version); + &cp_entry_1, version); if (err) return NULL; @@ -1208,19 +1190,19 @@ static struct folio *validate_checkpoint(struct f2fs_sb_info *sbi, cp_addr += cp_blocks - 1; err = get_checkpoint_version(sbi, cp_addr, &cp_block, - &cp_folio_2, version); + &cp_entry_2, version); if (err) goto invalid_cp; cur_version = *version; if (cur_version == pre_version) { *version = cur_version; - f2fs_folio_put(cp_folio_2, true); - return cp_folio_1; + f2fs_put_cache(cp_entry_2, true); + return cp_entry_1; } - f2fs_folio_put(cp_folio_2, true); + f2fs_put_cache(cp_entry_2, true); invalid_cp: - f2fs_folio_put(cp_folio_1, true); + f2fs_put_cache(cp_entry_1, true); return NULL; } @@ -1228,7 +1210,7 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi) { struct f2fs_checkpoint *cp_block; struct f2fs_super_block *fsb = sbi->raw_super; - struct folio *cp1, *cp2, *cur_folio; + struct f2fs_cached_block *cp1 = NULL, *cp2 = NULL, *cur_entry; unsigned long blk_size = sbi->blocksize; unsigned long long cp1_version = 0, cp2_version = 0; unsigned long long cp_start_blk_no; @@ -1255,22 +1237,22 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi) if (cp1 && cp2) { if (ver_after(cp2_version, cp1_version)) - cur_folio = cp2; + cur_entry = cp2; else - cur_folio = cp1; + cur_entry = cp1; } else if (cp1) { - cur_folio = cp1; + cur_entry = cp1; } else if (cp2) { - cur_folio = cp2; + cur_entry = cp2; } else { err = -EFSCORRUPTED; goto fail_no_cp; } - cp_block = folio_address(cur_folio); + cp_block = cache_address(cur_entry); memcpy(sbi->ckpt, cp_block, blk_size); - if (cur_folio == cp1) + if (cur_entry == cp1) sbi->cur_cp_pack = 1; else sbi->cur_cp_pack = 2; @@ -1285,30 +1267,31 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi) goto done; cp_blk_no = le32_to_cpu(fsb->cp_blkaddr); - if (cur_folio == cp2) + if (cur_entry == cp2) cp_blk_no += BIT(le32_to_cpu(fsb->log_blocks_per_seg)); for (i = 1; i < cp_blks; i++) { + struct f2fs_cached_block *cur_entry_payload; void *sit_bitmap_ptr; unsigned char *ckpt = (unsigned char *)sbi->ckpt; - cur_folio = f2fs_get_meta_folio(sbi, cp_blk_no + i); - if (IS_ERR(cur_folio)) { - err = PTR_ERR(cur_folio); + cur_entry_payload = f2fs_get_meta_cache(sbi, cp_blk_no + i); + if (IS_ERR(cur_entry_payload)) { + err = PTR_ERR(cur_entry_payload); goto free_fail_no_cp; } - sit_bitmap_ptr = folio_address(cur_folio); + sit_bitmap_ptr = cache_address(cur_entry_payload); memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size); - f2fs_folio_put(cur_folio, true); + f2fs_put_cache(cur_entry_payload, true); } done: - f2fs_folio_put(cp1, true); - f2fs_folio_put(cp2, true); + f2fs_put_cache(cp1, true); + f2fs_put_cache(cp2, true); return 0; free_fail_no_cp: - f2fs_folio_put(cp1, true); - f2fs_folio_put(cp2, true); + f2fs_put_cache(cp1, true); + f2fs_put_cache(cp2, true); fail_no_cp: kvfree(sbi->ckpt); return err; @@ -1384,12 +1367,12 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, unsigned long ino = 0; trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir, - get_pages(sbi, is_dir ? + get_nr_caches(sbi, is_dir ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); retry: if (unlikely(f2fs_cp_error(sbi))) { trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir, - get_pages(sbi, is_dir ? + get_nr_caches(sbi, is_dir ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); return -EIO; } @@ -1400,11 +1383,12 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, if (list_empty(head)) { spin_unlock(&sbi->inode_lock[type]); trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir, - get_pages(sbi, is_dir ? + get_nr_caches(sbi, is_dir ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); return 0; } fi = list_first_entry(head, struct f2fs_inode_info, dirty_list); + list_move_tail(&fi->dirty_list, head); inode = igrab(&fi->vfs_inode); spin_unlock(&sbi->inode_lock[type]); if (inode) { @@ -1427,11 +1411,6 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, else ino = cur_ino; } else { - /* - * We should submit bio, since it exists several - * writebacking dentry pages in the freeing inode. - */ - f2fs_submit_merged_write(sbi, DATA); cond_resched(); } goto retry; @@ -1442,7 +1421,7 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi) struct list_head *head = &sbi->inode_list[DIRTY_META]; struct inode *inode; struct f2fs_inode_info *fi; - s64 total = get_pages(sbi, F2FS_DIRTY_IMETA); + s64 total = get_nr_caches(sbi, F2FS_DIRTY_IMETA); while (total--) { if (unlikely(f2fs_cp_error(sbi))) @@ -1455,6 +1434,7 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi) } fi = list_first_entry(head, struct f2fs_inode_info, gdirty_list); + list_move_tail(&fi->gdirty_list, head); inode = igrab(&fi->vfs_inode); spin_unlock(&sbi->inode_lock[DIRTY_META]); if (inode) { @@ -1462,8 +1442,10 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi) /* it's on eviction */ if (is_inode_flag_set(inode, FI_DIRTY_INODE)) - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); iput(inode); + } else { + cond_resched(); } } return 0; @@ -1503,7 +1485,7 @@ static bool __need_flush_quota(struct f2fs_sb_info *sbi) } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) { clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH); ret = true; - } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) { + } else if (get_nr_caches(sbi, F2FS_DIRTY_QDATA)) { ret = true; } f2fs_up_write(&sbi->quota_sem); @@ -1515,14 +1497,10 @@ static bool __need_flush_quota(struct f2fs_sb_info *sbi) */ static int block_operations(struct f2fs_sb_info *sbi) { - struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, - .nr_to_write = LONG_MAX, - }; int err = 0, cnt = 0; /* - * Let's flush inline_data in dirty node pages. + * Let's flush inline_data in dirty node caches. */ f2fs_flush_inline_data(sbi); @@ -1551,7 +1529,7 @@ static int block_operations(struct f2fs_sb_info *sbi) retry_flush_dents: /* write all the dirty dentry pages */ - if (get_pages(sbi, F2FS_DIRTY_DENTS)) { + if (get_nr_caches(sbi, F2FS_DIRTY_DENTS)) { f2fs_unlock_all(sbi); err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true); if (err) @@ -1561,12 +1539,12 @@ static int block_operations(struct f2fs_sb_info *sbi) } /* - * POR: we should ensure that there are no dirty node pages + * POR: we should ensure that there are no dirty node caches * until finishing nat/sit flush. inode->i_blocks can be updated. */ f2fs_down_write(&sbi->node_change); - if (get_pages(sbi, F2FS_DIRTY_IMETA)) { + if (get_nr_caches(sbi, F2FS_DIRTY_IMETA)) { f2fs_up_write(&sbi->node_change); f2fs_unlock_all(sbi); err = f2fs_sync_inode_meta(sbi); @@ -1579,10 +1557,11 @@ static int block_operations(struct f2fs_sb_info *sbi) retry_flush_nodes: f2fs_down_write(&sbi->node_write); - if (get_pages(sbi, F2FS_DIRTY_NODES)) { + if (get_nr_caches(sbi, F2FS_DIRTY_NODES)) { f2fs_up_write(&sbi->node_write); atomic_inc(&sbi->wb_sync_req[NODE]); - err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO); + err = f2fs_writeback_node_caches(sbi, LONG_MAX, + true, false, FS_CP_NODE_IO, NULL); atomic_dec(&sbi->wb_sync_req[NODE]); if (err) { f2fs_up_write(&sbi->node_change); @@ -1608,12 +1587,12 @@ static void unblock_operations(struct f2fs_sb_info *sbi) f2fs_unlock_all(sbi); } -void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type) +void f2fs_sync_dirty_data(struct f2fs_sb_info *sbi, int type) { DEFINE_WAIT(wait); for (;;) { - if (!get_pages(sbi, type)) + if (!get_nr_caches(sbi, type)) break; if (unlikely(f2fs_cp_error(sbi) && @@ -1621,7 +1600,7 @@ void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type) break; if (type == F2FS_DIRTY_META) - f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_META_IO); else if (type == F2FS_WB_CP_DATA) f2fs_submit_merged_write(sbi, DATA); @@ -1700,31 +1679,24 @@ static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc) static void commit_checkpoint(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) { - struct writeback_control wbc = {}; + struct f2fs_cached_block *entry = f2fs_grab_meta_cache(sbi, blk_addr); - /* - * filemap_get_folios_tag and folio_lock again will take - * some extra time. Therefore, f2fs_update_meta_pages and - * f2fs_sync_meta_pages are combined in this function. - */ - struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); + memcpy(cache_address(entry), src, F2FS_BLKSIZE(sbi)); - memcpy(folio_address(folio), src, PAGE_SIZE); - - folio_mark_dirty(folio); - if (unlikely(!folio_clear_dirty_for_io(folio))) + f2fs_mark_cache_dirty(entry); + if (unlikely(!f2fs_cache_test_and_clear_dirty(entry))) f2fs_bug_on(sbi, 1); - /* writeout cp pack 2 page */ - if (unlikely(!__f2fs_write_meta_folio(folio, &wbc, FS_CP_META_IO))) { + /* writeout cp pack 2 cache */ + if (unlikely(!__f2fs_write_meta_cache(entry, FS_CP_META_IO))) { if (f2fs_cp_error(sbi)) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return; } f2fs_bug_on(sbi, true); } - f2fs_folio_put(folio, false); + f2fs_put_cache(entry, false); /* submit checkpoint (with barrier if NOBARRIER is not set) */ f2fs_submit_merged_write(sbi, META_FLUSH); @@ -1792,8 +1764,8 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) u64 kbytes_written; int err; - /* Flush all the NAT/SIT pages */ - f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + /* Flush all the NAT/SIT caches */ + f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_CP_META_IO); stat_cp_time(cpc, CP_TIME_SYNC_META); @@ -1816,7 +1788,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) } /* 2 cp + n data seg summary + orphan inode blocks */ - data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false); + data_sum_blocks = f2fs_nblocks_for_summary_flush(sbi, false); spin_lock_irqsave(&sbi->cp_lock, flags); if (data_sum_blocks < NR_CURSEG_DATA_TYPE) __set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); @@ -1824,7 +1796,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) __clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); spin_unlock_irqrestore(&sbi->cp_lock, flags); - orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num); + orphan_blocks = GET_ORPHAN_BLOCKS(sbi, orphan_num); ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks + orphan_blocks); @@ -1844,7 +1816,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP)); get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP)); - crc32 = f2fs_checkpoint_chksum(ckpt); + crc32 = f2fs_checkpoint_chksum(sbi, ckpt); *((__le32 *)((unsigned char *)ckpt + le32_to_cpu(ckpt->checksum_offset))) = cpu_to_le32(crc32); @@ -1861,16 +1833,16 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) blk = start_blk + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks; for (i = 0; i < nm_i->nat_bits_blocks; i++) - f2fs_update_meta_page(sbi, nm_i->nat_bits + - F2FS_BLK_TO_BYTES(i), blk + i); + f2fs_update_meta_block(sbi, nm_i->nat_bits + + F2FS_BLK_TO_BYTES(sbi, i), blk + i); } /* write out checkpoint buffer at block 0 */ - f2fs_update_meta_page(sbi, ckpt, start_blk++); + f2fs_update_meta_block(sbi, ckpt, start_blk++); for (i = 1; i < 1 + cp_payload_blks; i++) - f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE, - start_blk++); + f2fs_update_meta_block(sbi, (char *)ckpt + + i * F2FS_BLKSIZE(sbi), start_blk++); if (orphan_num) { write_orphan_inodes(sbi, start_blk); @@ -1891,16 +1863,16 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) start_blk += NR_CURSEG_NODE_TYPE; } - /* Here, we have one bio having CP pack except cp pack 2 page */ - f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + /* Here, we have one bio having CP pack except cp pack 2 blocks */ + f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_CP_META_IO); stat_cp_time(cpc, CP_TIME_SYNC_CP_META); - /* Wait for all dirty meta pages to be submitted for IO */ - f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META); + /* Wait for all dirty meta blocks to be submitted for IO */ + f2fs_sync_dirty_data(sbi, F2FS_DIRTY_META); stat_cp_time(cpc, CP_TIME_WAIT_DIRTY_META); - /* wait for previous submitted meta pages writeback */ - f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + /* wait for previous submitted meta blocks writeback */ + f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA); stat_cp_time(cpc, CP_TIME_WAIT_CP_DATA); /* flush all device cache */ @@ -1909,20 +1881,20 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) return err; stat_cp_time(cpc, CP_TIME_FLUSH_DEVICE); - /* barrier and flush checkpoint cp pack 2 page if it can */ + /* barrier and flush checkpoint cp pack 2 blocks if it can */ commit_checkpoint(sbi, ckpt, start_blk); - f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA); stat_cp_time(cpc, CP_TIME_WAIT_LAST_CP); /* - * invalidate intermediate page cache borrowed from meta inode which are - * used for migration of encrypted, verity or compressed inode's blocks. + * invalidate intermediate meta cache used for migration of + * encrypted, verity or compressed inode's blocks. */ if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) || - f2fs_sb_has_compression(sbi)) - f2fs_bug_on(sbi, - invalidate_inode_pages2_range(META_MAPPING(sbi), - MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1)); + f2fs_sb_has_compression(sbi)) { + f2fs_truncate_meta_caches(sbi, MAIN_BLKADDR(sbi), + MAX_BLKADDR(sbi) - MAIN_BLKADDR(sbi)); + } f2fs_release_ino_entry(sbi, false); @@ -1939,14 +1911,14 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) __set_cp_next_pack(sbi); /* - * redirty superblock if metadata like node page or inode cache is + * redirty superblock if metadata like node caches or inode cache is * updated during writing checkpoint. */ - if (get_pages(sbi, F2FS_DIRTY_NODES) || - get_pages(sbi, F2FS_DIRTY_IMETA)) + if (get_nr_caches(sbi, F2FS_DIRTY_NODES) || + get_nr_caches(sbi, F2FS_DIRTY_IMETA)) set_sbi_flag(sbi, SBI_IS_DIRTY); - f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS)); + f2fs_bug_on(sbi, get_nr_caches(sbi, F2FS_DIRTY_DENTS)); return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0; } @@ -2080,7 +2052,7 @@ void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi) sbi->max_orphans = (BLKS_PER_SEG(sbi) - F2FS_CP_PACKS - NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) * - F2FS_ORPHANS_PER_BLOCK; + F2FS_ORPHANS_PER_BLOCK(sbi); } int __init f2fs_create_checkpoint_caches(void) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 09d9b8d0fdcc..912c63bfa82b 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -807,7 +807,7 @@ void f2fs_end_read_compressed_page(struct folio *folio, bool failed, struct decompress_io_ctx *dic = folio->private; struct f2fs_sb_info *sbi = dic->sbi; - dec_page_count(sbi, F2FS_RD_DATA); + dec_cache_count(sbi, F2FS_RD_DATA); if (failed) WRITE_ONCE(dic->failed, true); @@ -909,7 +909,7 @@ bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) } for (i = 1, count = 1; i < cluster_size; i++, count++) { - block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node + i); /* [COMPR_ADDR, ..., COMPR_ADDR] */ @@ -950,7 +950,7 @@ static int __f2fs_get_cluster_blocks(struct inode *inode, int count, i; for (i = 0, count = 0; i < cluster_size; i++) { - block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node + i); if (__is_valid_data_blkaddr(blkaddr)) @@ -1152,11 +1152,11 @@ static int prepare_compress_overwrite(struct compress_ctx *cc, goto release_and_retry; } - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); f2fs_compress_ctx_add_page(cc, folio); if (!folio_test_uptodate(folio)) { - f2fs_handle_page_eio(sbi, folio, DATA); + f2fs_handle_page_eio(sbi, folio->index, DATA); release_and_retry: f2fs_put_rpages(cc); f2fs_unlock_rpages(cc, i + 1); @@ -1228,6 +1228,7 @@ int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) int i; int err; +repeat: err = f2fs_is_compressed_cluster(inode, start_idx); if (err < 0) return err; @@ -1239,12 +1240,11 @@ int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) /* truncate compressed cluster */ err = f2fs_prepare_compress_overwrite(inode, &pagep, start_idx, &fsdata); - - /* should not be a normal cluster */ - f2fs_bug_on(F2FS_I_SB(inode), err == 0); - - if (err <= 0) + if (err < 0) return err; + else if (err == 0) + /* the cluster became non-compressed one due to race case */ + goto repeat; rpages = fsdata; @@ -1328,7 +1328,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, goto out_unlock_op; for (i = 0; i < cc->cluster_size; i++) { - if (data_blkaddr(dn.inode, dn.node_folio, + if (data_blkaddr(dn.inode, dn.node_entry, dn.ofs_in_node + i) == NULL_ADDR) goto out_put_dnode; } @@ -1360,10 +1360,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, page_folio(cc->rpages[i + 1])->index, cic); fio.compressed_page = cc->cpages[i]; - fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio, + fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_entry, dn.ofs_in_node + i + 1); - /* wait for GCed page writeback via META_MAPPING */ + /* wait for GCed page writeback via generic cache */ f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); } @@ -1477,7 +1477,7 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) f2fs_compress_free_page(page); if (atomic_dec_return(&cic->pending_pages)) { - dec_page_count(sbi, type); + dec_cache_count(sbi, type); return; } @@ -1494,12 +1494,12 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) kmem_cache_free(cic_entry_slab, cic); /* - * Make sure dec_page_count() is the last access to sbi. + * Make sure dec_cache_count() is the last access to sbi. * Once it drops the F2FS_WB_CP_DATA counter to zero, the * unmount thread can proceed to destroy sbi and * sbi->page_array_slab. */ - dec_page_count(sbi, type); + dec_cache_count(sbi, type); } static int f2fs_write_raw_pages(struct compress_ctx *cc, @@ -1551,7 +1551,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc, if (folio_test_writeback(folio)) { if (wbc->sync_mode == WB_SYNC_NONE) goto continue_unlock; - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); } if (!folio_clear_dirty_for_io(folio)) @@ -1883,14 +1883,14 @@ void f2fs_put_folio_dic(struct folio *folio, bool in_task) unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, unsigned int ofs_in_node) { - bool compressed = data_blkaddr(dn->inode, dn->node_folio, + bool compressed = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node) == COMPRESS_ADDR; int i = compressed ? 1 : 0; - block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, + block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node + i); for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { - block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node + i); if (!__is_valid_data_blkaddr(blkaddr)) @@ -1902,30 +1902,19 @@ unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, return compressed ? i - 1 : i; } -const struct address_space_operations f2fs_compress_aops = { - .release_folio = f2fs_release_folio, - .invalidate_folio = f2fs_invalidate_folio, - .migrate_folio = filemap_migrate_folio, -}; - -struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) -{ - return sbi->compress_inode->i_mapping; -} - void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, block_t blkaddr, unsigned int len) { - if (!sbi->compress_inode) + if (!test_opt(sbi, COMPRESS_CACHE)) return; - invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1); + + f2fs_drop_cache_range(COMPRESS_CACHE(sbi), blkaddr, len, false); } static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct folio *folio, nid_t ino, block_t blkaddr) { - struct folio *cfolio; - int ret; + struct f2fs_cached_block *entry; if (!test_opt(sbi, COMPRESS_CACHE)) return; @@ -1933,52 +1922,66 @@ static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) return; - if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) + if (!f2fs_available_free_memory(sbi, COMPRESS_BLOCK)) return; - cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr); - if (!IS_ERR(cfolio)) { - f2fs_folio_put(cfolio, false); + entry = f2fs_find_cache(COMPRESS_CACHE(sbi), blkaddr, + F2FS_CACHE_ACCESS); + if (!IS_ERR(entry)) { + f2fs_put_cache(entry, false); return; } - cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL); - if (!cfolio) + entry = f2fs_grab_cache(COMPRESS_CACHE(sbi), blkaddr, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(entry)) return; - ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio, - blkaddr, GFP_NOFS); - if (ret) { - f2fs_folio_put(cfolio, false); + /* someone else has populated the entry after f2fs_find_cache() */ + if (f2fs_cache_test_uptodate(entry)) { + /* + * a block address can only be owned by one inode, if the + * entry was cached by another inode, one of copies must be + * stale, drop it and let readers load the block from disk. + */ + if (unlikely(entry->ino != ino)) { + f2fs_bug_on(sbi, 1); + f2fs_warn_ratelimited(sbi, + "%s: blkaddr %u was cached by ino %u, current ino %u", + __func__, blkaddr, entry->ino, ino); + f2fs_truncate_locked_cache(entry, false); + } + f2fs_put_cache(entry, true); return; } - folio_set_f2fs_data(cfolio, ino); - - memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE); - folio_mark_uptodate(cfolio); - f2fs_folio_put(cfolio, true); + entry->ino = ino; + memcpy(cache_address(entry), folio_address(folio), sbi->blocksize); + f2fs_cache_set_uptodate(entry); + f2fs_put_cache(entry, true); } bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, block_t blkaddr) { - struct folio *cfolio; + struct f2fs_cached_block *entry; bool hitted = false; if (!test_opt(sbi, COMPRESS_CACHE)) return false; - cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi), - blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); - if (!IS_ERR(cfolio)) { - if (folio_test_uptodate(cfolio)) { + entry = f2fs_find_cache(COMPRESS_CACHE(sbi), blkaddr, + F2FS_CACHE_ACCESS); + if (!IS_ERR(entry)) { + f2fs_lock_cache(entry); + if (f2fs_is_compress_cache(entry) && + f2fs_cache_test_uptodate(entry)) { atomic_inc(&sbi->compress_page_hit); memcpy(folio_address(folio), - folio_address(cfolio), folio_size(folio)); + cache_address(entry), folio_size(folio)); hitted = true; } - f2fs_folio_put(cfolio, true); + f2fs_put_cache(entry, true); } return hitted; @@ -1986,71 +1989,49 @@ bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) { - struct address_space *mapping = COMPRESS_MAPPING(sbi); - struct folio_batch fbatch; - pgoff_t index = 0; - pgoff_t end = MAX_BLKADDR(sbi); + struct f2fs_cached_block_list *cache = COMPRESS_CACHE(sbi); + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; + pgoff_t index = 0, end = ULONG_MAX; + int nr; + int i; - if (!mapping->nrpages) + if (!cache->num_entries) return; +next: + nr = f2fs_cache_gang_lookup(cache, entries, &index, end); + if (!nr) + return; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; - folio_batch_init(&fbatch); + index = entry->index + 1; - do { - unsigned int nr, i; + f2fs_lock_cache(entry); + if (unlikely(!f2fs_is_compress_cache(entry))) + goto unlock; + if (entry->ino != ino) + goto unlock; - nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); - if (!nr) - break; + f2fs_truncate_locked_cache(entry, false); +unlock: + f2fs_unlock_cache(entry); + } + f2fs_cache_gang_release(entries, nr); - for (i = 0; i < nr; i++) { - struct folio *folio = fbatch.folios[i]; - - folio_lock(folio); - if (folio->mapping != mapping) { - folio_unlock(folio); - continue; - } - - if (ino != folio_get_f2fs_data(folio)) { - folio_unlock(folio); - continue; - } - - generic_error_remove_folio(mapping, folio); - folio_unlock(folio); - } - folio_batch_release(&fbatch); + if (index < end) { cond_resched(); - } while (index < end); + goto next; + } } -int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) +void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi) { - struct inode *inode; - if (!test_opt(sbi, COMPRESS_CACHE)) - return 0; - - inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); - if (IS_ERR(inode)) - return PTR_ERR(inode); - sbi->compress_inode = inode; + return; sbi->compress_percent = COMPRESS_PERCENT; sbi->compress_watermark = COMPRESS_WATERMARK; - atomic_set(&sbi->compress_page_hit, 0); - - return 0; -} - -void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) -{ - if (!sbi->compress_inode) - return; - iput(sbi->compress_inode); - sbi->compress_inode = NULL; } int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index ca8232a9095f..28fd8afff5d4 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -41,11 +41,6 @@ struct f2fs_folio_state { unsigned int read_pages_pending; }; -struct f2fs_bio { - struct work_struct work; - struct bio bio; -}; - #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void) @@ -63,14 +58,10 @@ bool f2fs_is_cp_guaranteed(const struct folio *folio) { struct address_space *mapping = folio->mapping; struct inode *inode; - struct f2fs_sb_info *sbi; inode = mapping->host; - sbi = F2FS_I_SB(inode); - if (inode->i_ino == F2FS_META_INO(sbi) || - inode->i_ino == F2FS_NODE_INO(sbi) || - S_ISDIR(inode->i_mode)) + if (S_ISDIR(inode->i_mode)) return true; if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || @@ -79,23 +70,6 @@ bool f2fs_is_cp_guaranteed(const struct folio *folio) return false; } -static enum count_type __read_io_type(struct folio *folio) -{ - struct address_space *mapping = folio->mapping; - - if (mapping) { - struct inode *inode = mapping->host; - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - - if (inode->i_ino == F2FS_META_INO(sbi)) - return F2FS_RD_META; - - if (inode->i_ino == F2FS_NODE_INO(sbi)) - return F2FS_RD_NODE; - } - return F2FS_RD_DATA; -} - /* postprocessing steps for read bios */ enum bio_post_read_step { #ifdef CONFIG_F2FS_FS_COMPRESSION @@ -169,13 +143,7 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) } while (nr_pages--) - dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); - - if (bio->bi_status == BLK_STS_OK && - F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && - f2fs_sanity_check_node_footer(F2FS_F_SB(folio), - folio, folio->index, NODE_TYPE_REGULAR, true)) - bio->bi_status = BLK_STS_IOERR; + dec_cache_count(F2FS_F_SB(folio), F2FS_RD_DATA); if (finished) folio_end_read(folio, bio->bi_status == BLK_STS_OK); @@ -303,7 +271,7 @@ static void f2fs_read_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); struct bio_post_read_ctx *ctx; - bool intask = in_task() && !irqs_disabled(); + bool intask = !bio_in_atomic(); iostat_update_and_unbind_ctx(bio); ctx = bio->bi_private; @@ -330,6 +298,40 @@ static void f2fs_read_end_io(struct bio *bio) f2fs_verify_and_finish_bio(bio, intask); } +static void f2fs_cache_write_end_bio(struct bio *bio) +{ + struct f2fs_cached_block *entry = F2FS_BIO(bio)->entry; + struct f2fs_sb_info *sbi = entry->cache->sbi; + struct f2fs_cached_block *next; + + if (bio->bi_status != BLK_STS_OK) + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); + + while (entry) { + next = entry->next_entry; + entry->next_entry = NULL; + + if (f2fs_is_node_cache(entry)) { + f2fs_sanity_check_node_footer(sbi, entry, + entry->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, entry->index != nid_of_node(sbi, entry)); + } + if (f2fs_in_warm_node_list(sbi, entry)) + f2fs_del_fsync_node_entry(sbi, entry); + + dec_cache_count(sbi, F2FS_WB_CP_DATA); + + if (!get_nr_caches(sbi, F2FS_WB_CP_DATA) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + + f2fs_end_cache_writeback(entry); + entry = next; + } + bio_put(bio); +} + static void f2fs_write_end_bio(struct bio *bio) { struct f2fs_sb_info *sbi = bio->bi_private; @@ -338,6 +340,11 @@ static void f2fs_write_end_bio(struct bio *bio) if (time_to_inject(sbi, FAULT_WRITE_IO)) bio->bi_status = BLK_STS_IOERR; + if (f2fs_is_cache_bio(bio)) { + f2fs_cache_write_end_bio(bio); + return; + } + bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; enum count_type type; @@ -359,21 +366,13 @@ static void f2fs_write_end_bio(struct bio *bio) } } - if (is_node_folio(folio)) { - f2fs_sanity_check_node_footer(sbi, folio, - folio->index, NODE_TYPE_REGULAR, true); - f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); - } - if (f2fs_in_warm_node_list(folio)) - f2fs_del_fsync_node_entry(sbi, folio); - - dec_page_count(sbi, type); + dec_cache_count(sbi, type); /* * we should access sbi before folio_end_writeback() to * avoid racing w/ kill_f2fs_super() */ - if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + if (type == F2FS_WB_CP_DATA && !get_nr_caches(sbi, type) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); @@ -399,7 +398,9 @@ static void f2fs_write_end_io(struct bio *bio) sbi = bio->bi_private; - if (in_atomic() && bio->bi_iter.bi_size > sbi->max_atc_write_bio_size) { + if (bio_in_atomic() && + (bio->bi_iter.bi_size > sbi->max_atc_write_bio_size || + F2FS_BIO(bio)->entry_cnt > sbi->max_atc_write_bio_entry_cnt)) { struct work_struct *w; w = &container_of(bio, struct f2fs_bio, bio)->work; @@ -410,6 +411,40 @@ static void f2fs_write_end_io(struct bio *bio) } } +static void f2fs_cache_read_end_io(struct bio *bio) +{ + struct f2fs_cached_block *entry = F2FS_BIO(bio)->entry; + struct f2fs_sb_info *sbi = entry->cache->sbi; + enum count_type io_type = IS_META_CACHE(entry->cache) ? + F2FS_RD_META : F2FS_RD_NODE; + struct f2fs_cached_block *next; + + iostat_update_and_unbind_ctx(bio); + + if (time_to_inject(sbi, FAULT_READ_IO)) + bio->bi_status = BLK_STS_IOERR; + + while (entry) { + next = entry->next_entry; + entry->next_entry = NULL; + + if (bio->bi_status == BLK_STS_OK && + f2fs_is_node_cache(entry) && + f2fs_sanity_check_node_footer(sbi, entry, + entry->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + + if (bio->bi_status == BLK_STS_OK) + f2fs_cache_set_uptodate(entry); + + dec_cache_count(sbi, io_type); + + f2fs_unlock_cache(entry); + entry = next; + } + bio_put(bio); +} + #ifdef CONFIG_BLK_DEV_ZONED static void f2fs_zone_write_end_io(struct bio *bio) { @@ -439,7 +474,7 @@ struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, } if (sector) - *sector = SECTOR_FROM_BLOCK(blk_addr); + *sector = SECTOR_FROM_BLOCK(sbi, blk_addr); return bdev; } @@ -500,13 +535,21 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) struct bio *bio; bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); + /* avoid reversed IO if bio is split due to exceeding max_segments */ + if (npages > 1 && !is_read_io(fio->op) && + f2fs_is_sequential_zone_area(sbi, fio->new_blkaddr)) + npages = min_t(int, npages, bdev_max_segments(bdev)); bio = bio_alloc_bioset(bdev, npages, fio->op | fio->op_flags | f2fs_io_flags(fio), GFP_NOIO, &f2fs_bioset); bio->bi_iter.bi_sector = sector; + F2FS_BIO(bio)->entry = NULL; + bio->bi_private = NULL; if (is_read_io(fio->op)) { - bio->bi_end_io = f2fs_read_end_io; - bio->bi_private = NULL; + if (fio->is_cache) + bio->bi_end_io = f2fs_cache_read_end_io; + else + bio->bi_end_io = f2fs_read_end_io; } else { bio->bi_end_io = f2fs_write_end_io; bio->bi_private = sbi; @@ -514,6 +557,7 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) fio->type, fio->temp); bio->bi_write_stream = f2fs_io_type_to_write_stream(bdev, fio->type, fio->temp); + F2FS_BIO(bio)->entry_cnt = 0; } iostat_alloc_and_bind_ctx(sbi, bio, NULL); @@ -532,7 +576,7 @@ static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ - if (!fio || !fio->encrypted_page) + if (!fio || (!fio->encrypted_page && !fio->is_cache)) fscrypt_set_bio_crypt_ctx(bio, inode, (loff_t)first_idx << inode->i_blkbits, gfp_mask); @@ -593,16 +637,19 @@ static void __submit_merged_bio(struct f2fs_bio_info *io) } static bool __has_merged_page(struct bio *bio, struct inode *inode, - struct folio *folio, nid_t ino) + struct folio *folio) { struct folio_iter fi; if (!bio) return false; - if (!inode && !folio && !ino) + if (!inode && !folio) return true; + if (f2fs_is_cache_bio(bio)) + return false; + bio_for_each_folio_all(fi, bio) { struct folio *target = fi.folio; @@ -616,8 +663,6 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode, return true; if (folio && folio == target) return true; - if (ino && ino == ino_of_node(target)) - return true; } return false; @@ -684,26 +729,25 @@ static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, f2fs_up_write_trace(&io->io_rwsem, &lc); } -static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, +static void __submit_merged_data_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, struct folio *folio, - nid_t ino, enum page_type type, bool writeback) + bool writeback) { enum temp_type temp; bool ret = true; - bool force = !inode && !folio && !ino; + bool force = !inode && !folio; for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { - if (!force) { - enum page_type btype = PAGE_TYPE_OF_BIO(type); - struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + if (!force) { + struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; struct f2fs_lock_context lc; f2fs_down_read_trace(&io->io_rwsem, &lc); - ret = __has_merged_page(io->bio, inode, folio, ino); + ret = __has_merged_page(io->bio, inode, folio); f2fs_up_read_trace(&io->io_rwsem, &lc); } if (ret) { - __f2fs_submit_merged_write(sbi, type, temp); + __f2fs_submit_merged_write(sbi, DATA, temp); /* * For waitting writebck case, if the bio owned by the * folio is already submitted, we do not need to submit @@ -712,29 +756,86 @@ static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, if (writeback) break; } + } +} + +void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, + struct inode *inode, struct folio *folio) +{ + __submit_merged_data_write_cond(sbi, inode, folio, false); +} + +void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, + struct folio *folio) +{ + __submit_merged_data_write_cond(sbi, NULL, folio, true); +} + +static bool __has_merged_cache(struct f2fs_sb_info *sbi, struct bio *bio, + struct f2fs_cached_block *target, nid_t ino) +{ + struct f2fs_cached_block *entry; + + if (!bio) + return false; + + entry = F2FS_BIO(bio)->entry; + + while (entry) { + if (target && entry == target) + return true; + + if (ino) { + if (likely(f2fs_is_node_cache(entry))) { + if (ino_of_node(sbi, entry) == ino) + return true; + } else { + WARN_ON_ONCE(1); + } + } + entry = entry->next_entry; + } + return false; +} + +bool f2fs_submit_merged_write_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, + nid_t ino, enum page_type type) +{ + enum temp_type temp; + bool ret = false; + bool force = !entry && !ino; + + for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { + enum page_type btype = PAGE_TYPE_OF_BIO(type); + struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + struct f2fs_lock_context lc; + bool merged = true; + + if (!force) { + f2fs_down_read_trace(&io->io_rwsem, &lc); + merged = __has_merged_cache(sbi, io->bio, entry, ino); + f2fs_up_read_trace(&io->io_rwsem, &lc); + } + + if (merged) { + __f2fs_submit_merged_write(sbi, type, temp); + ret = true; + } /* TODO: use HOT temp only for meta pages now. */ if (type >= META) break; } + return ret; } void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) { - __submit_merged_write_cond(sbi, NULL, NULL, 0, type, false); -} - -void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, - struct inode *inode, struct folio *folio, - nid_t ino, enum page_type type) -{ - __submit_merged_write_cond(sbi, inode, folio, ino, type, false); -} - -void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, - struct folio *folio, enum page_type type) -{ - __submit_merged_write_cond(sbi, NULL, folio, 0, type, true); + if (type == DATA) + __submit_merged_data_write_cond(sbi, NULL, NULL, false); + else + f2fs_submit_merged_write_cache(sbi, NULL, 0, type); } void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) @@ -772,8 +873,8 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) if (fio->io_wbc && !is_read_io(fio->op)) wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE); - inc_page_count(fio->sbi, is_read_io(fio->op) ? - __read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false)); + inc_cache_count(fio->sbi, is_read_io(fio->op) ? + F2FS_RD_DATA : WB_DATA_TYPE(fio->folio, false)); if (is_read_io(bio_op(bio))) f2fs_submit_read_bio(fio->sbi, bio, fio->type); @@ -790,7 +891,19 @@ static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, return false; if (last_blkaddr + 1 != cur_blkaddr) return false; - return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); + if (bio->bi_bdev != f2fs_target_device(sbi, cur_blkaddr, NULL)) + return false; + /* avoid reversed IO if bio is split due to exceeding max_segments/sectors */ + if (f2fs_is_sequential_zone_area(sbi, cur_blkaddr)) { + struct request_queue *q = bdev_get_queue(bio->bi_bdev); + + if (bio->bi_vcnt >= queue_max_segments(q)) + return false; + if (bio_sectors(bio) + (F2FS_BLKSIZE(sbi) >> SECTOR_SHIFT) > + queue_max_sectors(q)) + return false; + } + return true; } static bool io_type_is_mergeable(struct f2fs_bio_info *io, @@ -800,6 +913,8 @@ static bool io_type_is_mergeable(struct f2fs_bio_info *io, if (io->fio.op != fio->op) return false; + if (io->fio.is_cache != fio->is_cache) + return false; return (io->fio.op_flags & mask) == (fio->op_flags & mask); } @@ -907,8 +1022,8 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, if (target) found = (target == be->bio); else - found = __has_merged_page(be->bio, NULL, - folio, 0); + found = __has_merged_page(be->bio, + NULL, folio); if (found) break; } @@ -924,8 +1039,8 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, if (target) found = (target == be->bio); else - found = __has_merged_page(be->bio, NULL, - folio, 0); + found = __has_merged_page(be->bio, + NULL, folio); if (found) { target = be->bio; del_bio_entry(be); @@ -1003,7 +1118,7 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio) if (fio->io_wbc) wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio)); - inc_page_count(fio->sbi, WB_DATA_TYPE(folio, false)); + inc_cache_count(fio->sbi, WB_DATA_TYPE(folio, false)); *fio->last_block = fio->new_blkaddr; *fio->bio = bio; @@ -1031,6 +1146,33 @@ static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) f2fs_blkz_is_seq(sbi, devi, blkaddr) && (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); } + +static void f2fs_wait_zone_io_completion(struct f2fs_sb_info *sbi, + struct f2fs_bio_info *io, enum page_type btype) +{ + if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { + wait_for_completion_io(&io->zone_wait); + bio_put(io->zone_pending_bio); + io->zone_pending_bio = NULL; + io->bi_private = NULL; + } +} + +static void f2fs_submit_zone_io(struct f2fs_sb_info *sbi, + struct f2fs_io_info *fio, struct f2fs_bio_info *io, + enum page_type btype) +{ + if (f2fs_sb_has_blkzoned(sbi) && btype < META && + is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { + bio_get(io->bio); + reinit_completion(&io->zone_wait); + io->bi_private = io->bio->bi_private; + io->bio->bi_private = io; + io->bio->bi_end_io = f2fs_zone_write_end_io; + io->zone_pending_bio = io->bio; + __submit_merged_bio(io); + } +} #endif void f2fs_submit_page_write(struct f2fs_io_info *fio) @@ -1047,14 +1189,8 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) f2fs_down_write_trace(&io->io_rwsem, &lc); next: #ifdef CONFIG_BLK_DEV_ZONED - if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { - wait_for_completion_io(&io->zone_wait); - bio_put(io->zone_pending_bio); - io->zone_pending_bio = NULL; - io->bi_private = NULL; - } + f2fs_wait_zone_io_completion(sbi, io, btype); #endif - if (fio->in_list) { spin_lock(&io->io_lock); if (list_empty(&io->io_list)) { @@ -1080,7 +1216,7 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) fio->submitted = 1; type = WB_DATA_TYPE(bio_folio, fio->compressed_page); - inc_page_count(sbi, type); + inc_cache_count(sbi, type); if (io->bio && (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, @@ -1101,6 +1237,8 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) goto alloc_new; } + F2FS_BIO(io->bio)->entry_cnt++; + if (fio->io_wbc) wbc_account_cgroup_owner(fio->io_wbc, fio->folio, folio_size(fio->folio)); @@ -1109,16 +1247,110 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) trace_f2fs_submit_folio_write(fio->folio, fio); #ifdef CONFIG_BLK_DEV_ZONED - if (f2fs_sb_has_blkzoned(sbi) && btype < META && - is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { - bio_get(io->bio); - reinit_completion(&io->zone_wait); - io->bi_private = io->bio->bi_private; - io->bio->bi_private = io; - io->bio->bi_end_io = f2fs_zone_write_end_io; - io->zone_pending_bio = io->bio; + f2fs_submit_zone_io(sbi, fio, io, btype); +#endif + + if (fio->in_list) + goto next; +out: + if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || + !f2fs_is_checkpoint_ready(sbi)) __submit_merged_bio(io); + f2fs_up_write_trace(&io->io_rwsem, &lc); +} + +static void f2fs_bio_add_cache(struct f2fs_io_info *fio, struct bio *bio) +{ + struct f2fs_bio *fbio = F2FS_BIO(bio); + struct f2fs_cached_block *head = fbio->entry; + struct f2fs_cached_block *new = fio->cache_entry; + + new->next_entry = head; + fbio->entry = new; +} + +int f2fs_submit_cache_read(struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + struct f2fs_cached_block *entry = fio->cache_entry; + struct bio *bio; + enum count_type io_type = IS_META_CACHE(entry->cache) ? + F2FS_RD_META : F2FS_RD_NODE; + + if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, + fio->is_por ? META_POR : (__is_meta_io(fio) ? + META_GENERIC : DATA_GENERIC_ENHANCE))) + return -EFSCORRUPTED; + + bio = __bio_alloc(fio, 1); + + bio_add_virt_nofail(bio, cache_address(entry), sbi->blocksize); + f2fs_bio_add_cache(fio, bio); + inc_cache_count(sbi, io_type); + + f2fs_submit_read_bio(sbi, bio, fio->type); + return 0; +} + +void f2fs_submit_cache_write(struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); + struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; + struct f2fs_lock_context lc; + struct folio *folio; + + f2fs_bug_on(sbi, is_read_io(fio->op)); + + f2fs_down_write_trace(&io->io_rwsem, &lc); +next: +#ifdef CONFIG_BLK_DEV_ZONED + f2fs_wait_zone_io_completion(sbi, io, btype); +#endif + if (fio->in_list) { + spin_lock(&io->io_lock); + if (list_empty(&io->io_list)) { + spin_unlock(&io->io_lock); + goto out; + } + fio = list_first_entry(&io->io_list, + struct f2fs_io_info, list); + list_del(&fio->list); + spin_unlock(&io->io_lock); } + + verify_fio_blkaddr(fio); + + fio->submitted = 1; + inc_cache_count(sbi, F2FS_WB_CP_DATA); + + if (io->bio && + (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, + fio->new_blkaddr))) + __submit_merged_bio(io); +alloc_new: + if (io->bio == NULL) { + io->bio = __bio_alloc(fio, BIO_MAX_VECS); + io->fio = *fio; + } + + folio = cache_folio(fio->cache_entry); + + if (!bio_add_folio(io->bio, folio, sbi->blocksize, + offset_in_folio(folio, cache_address(fio->cache_entry)))) { + f2fs_bug_on(sbi, !F2FS_BIO(io->bio)->entry); + + __submit_merged_bio(io); + goto alloc_new; + } + + f2fs_bio_add_cache(fio, io->bio); + F2FS_BIO(io->bio)->entry_cnt++; + + io->last_block_in_bio = fio->new_blkaddr; + +#ifdef CONFIG_BLK_DEV_ZONED + f2fs_submit_zone_io(sbi, fio, io, btype); #endif if (fio->in_list) goto next; @@ -1185,20 +1417,20 @@ static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi, bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index, for_write); - /* wait for GCed page writeback via META_MAPPING */ + /* wait for GCed page writeback via generic cache */ f2fs_wait_on_block_writeback(inode, blkaddr); if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) f2fs_bug_on(sbi, 1); - inc_page_count(sbi, F2FS_RD_DATA); - f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); + inc_cache_count(sbi, F2FS_RD_DATA); + f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE(sbi)); f2fs_submit_read_bio(sbi, bio, DATA); } static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { - __le32 *addr = get_dnode_addr(dn->inode, dn->node_folio); + __le32 *addr = get_dnode_addr(dn->inode, dn->node_entry); dn->data_blkaddr = blkaddr; addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); @@ -1212,9 +1444,9 @@ static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) */ void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { - f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); + f2fs_cache_wait_writeback(dn->node_entry); __set_data_blkaddr(dn, blkaddr); - if (folio_mark_dirty(dn->node_folio)) + if (f2fs_mark_cache_dirty(dn->node_entry)) dn->node_changed = true; } @@ -1242,7 +1474,7 @@ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, dn->ofs_in_node, count); - f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); + f2fs_cache_wait_writeback(dn->node_entry); for (; count > 0; dn->ofs_in_node++) { block_t blkaddr = f2fs_data_blkaddr(dn); @@ -1253,7 +1485,7 @@ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) } } - if (folio_mark_dirty(dn->node_folio)) + if (f2fs_mark_cache_dirty(dn->node_entry)) dn->node_changed = true; return 0; } @@ -1271,7 +1503,7 @@ int f2fs_reserve_new_block(struct dnode_of_data *dn) int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) { - bool need_put = dn->inode_folio ? false : true; + bool need_put = dn->inode_entry ? false : true; int err; err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); @@ -1437,11 +1669,11 @@ struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, * * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and * f2fs_unlock_op(). - * Note that, ifolio is set only by make_empty_dir, and if any error occur, - * ifolio should be released by this function. + * Note that, ientry is set only by make_empty_dir, and if any error occur, + * ientry should be released by this function. */ struct folio *f2fs_get_new_data_folio(struct inode *inode, - struct folio *ifolio, pgoff_t index, bool new_i_size) + struct f2fs_cached_block *ientry, pgoff_t index, bool new_i_size) { struct address_space *mapping = inode->i_mapping; struct folio *folio; @@ -1451,20 +1683,20 @@ struct folio *f2fs_get_new_data_folio(struct inode *inode, folio = f2fs_grab_cache_folio(mapping, index, true); if (IS_ERR(folio)) { /* - * before exiting, we should make sure ifolio will be released + * before exiting, we should make sure ientry will be released * if any error occur. */ - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return ERR_PTR(-ENOMEM); } - set_new_dnode(&dn, inode, ifolio, NULL, 0); + set_new_dnode(&dn, inode, ientry, NULL, 0); err = f2fs_reserve_block(&dn, index); if (err) { f2fs_folio_put(folio, true); return ERR_PTR(err); } - if (!ifolio) + if (!ientry) f2fs_put_dnode(&dn); if (folio_test_uptodate(folio)) @@ -1477,8 +1709,8 @@ struct folio *f2fs_get_new_data_folio(struct inode *inode, } else { f2fs_folio_put(folio, true); - /* if ifolio exists, blkaddr should be NEW_ADDR */ - f2fs_bug_on(F2FS_I_SB(inode), ifolio); + /* if ientry exists, blkaddr should be NEW_ADDR */ + f2fs_bug_on(F2FS_I_SB(inode), ientry); folio = f2fs_get_lock_data_folio(inode, index, true); if (IS_ERR(folio)) return folio; @@ -1515,7 +1747,7 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); old_blkaddr = dn->data_blkaddr; - err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, + err = f2fs_allocate_data_block(sbi, old_blkaddr, &dn->data_blkaddr, &sum, seg_type, NULL); if (err) { if (old_blkaddr == NULL_ADDR) @@ -1727,7 +1959,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) start_pgofs = pgofs; prealloc = 0; last_ofs_in_node = ofs_in_node = dn.ofs_in_node; - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); next_block: blkaddr = f2fs_data_blkaddr(&dn); @@ -1942,12 +2174,12 @@ static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, if (pos + len > i_size_read(inode)) return false; - map.m_lblk = F2FS_BYTES_TO_BLK(pos); + map.m_lblk = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode), pos); map.m_next_pgofs = NULL; map.m_next_extent = NULL; map.m_seg_type = NO_CHECK_TYPE; map.m_may_create = false; - last_lblk = F2FS_BLK_ALIGN(pos + len); + last_lblk = F2FS_BLK_ALIGN(F2FS_I_SB(inode), pos + len); while (map.m_lblk < last_lblk) { map.m_len = last_lblk - map.m_lblk; @@ -1978,27 +2210,27 @@ static int f2fs_xattr_fiemap(struct inode *inode, if (f2fs_has_inline_xattr(inode)) { int offset; - struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), - inode->i_ino, false); + struct f2fs_cached_block *entry = + f2fs_grab_node_cache(sbi, inode->i_ino); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(entry)) + return PTR_ERR(entry); err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return err; } - phys = F2FS_BLK_TO_BYTES(ni.blk_addr); + phys = F2FS_BLK_TO_BYTES(sbi, ni.blk_addr); offset = offsetof(struct f2fs_inode, i_addr) + - sizeof(__le32) * (DEF_ADDRS_PER_INODE - + sizeof(__le32) * (DEF_ADDRS_PER_INODE(sbi) - get_inline_xattr_addrs(inode)); phys += offset; len = inline_xattr_size(inode); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; @@ -2012,22 +2244,22 @@ static int f2fs_xattr_fiemap(struct inode *inode, } if (xnid) { - struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), - xnid, false); + struct f2fs_cached_block *entry = + f2fs_grab_node_cache(sbi, xnid); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(entry)) + return PTR_ERR(entry); err = f2fs_get_node_info(sbi, xnid, &ni, false); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return err; } - phys = F2FS_BLK_TO_BYTES(ni.blk_addr); + phys = F2FS_BLK_TO_BYTES(sbi, ni.blk_addr); len = inode->i_sb->s_blocksize; - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); flags = FIEMAP_EXTENT_LAST; } @@ -2043,6 +2275,7 @@ static int f2fs_xattr_fiemap(struct inode *inode, int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_map_blocks map; sector_t start_blk, last_blk, blk_len, max_len; pgoff_t next_pgofs; @@ -2066,7 +2299,7 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock_shared(inode); - maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + maxbytes = F2FS_BLK_TO_BYTES(sbi, max_file_blocks(sbi, inode)); if (start > maxbytes) { ret = -EFBIG; goto out; @@ -2086,10 +2319,10 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, goto out; } - start_blk = F2FS_BYTES_TO_BLK(start); - last_blk = F2FS_BYTES_TO_BLK(start + len - 1); + start_blk = F2FS_BYTES_TO_BLK(sbi, start); + last_blk = F2FS_BYTES_TO_BLK(sbi, start + len - 1); blk_len = last_blk - start_blk + 1; - max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk; + max_len = F2FS_BYTES_TO_BLK(sbi, maxbytes) - start_blk; next: memset(&map, 0, sizeof(map)); @@ -2111,7 +2344,7 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { start_blk = next_pgofs; - if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes) + if (F2FS_BLK_TO_BYTES(sbi, start_blk) < maxbytes) goto prep_next; flags |= FIEMAP_EXTENT_LAST; @@ -2159,14 +2392,14 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, } else if (compr_appended) { unsigned int appended_blks = cluster_size - count_in_cluster + 1; - size += F2FS_BLK_TO_BYTES(appended_blks); + size += F2FS_BLK_TO_BYTES(sbi, appended_blks); start_blk += appended_blks; compr_cluster = false; } else { - logical = F2FS_BLK_TO_BYTES(start_blk); + logical = F2FS_BLK_TO_BYTES(sbi, start_blk); phys = __is_valid_data_blkaddr(map.m_pblk) ? - F2FS_BLK_TO_BYTES(map.m_pblk) : 0; - size = F2FS_BLK_TO_BYTES(map.m_len); + F2FS_BLK_TO_BYTES(sbi, map.m_pblk) : 0; + size = F2FS_BLK_TO_BYTES(sbi, map.m_len); flags = 0; if (compr_cluster) { @@ -2174,13 +2407,13 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, count_in_cluster += map.m_len; if (count_in_cluster == cluster_size) { compr_cluster = false; - size += F2FS_BLKSIZE; + size += F2FS_BLKSIZE(sbi); } } else if (map.m_flags & F2FS_MAP_DELALLOC) { flags = FIEMAP_EXTENT_UNWRITTEN; } - start_blk += F2FS_BYTES_TO_BLK(size); + start_blk += F2FS_BYTES_TO_BLK(sbi, size); } prep_next: @@ -2200,7 +2433,8 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, static inline loff_t f2fs_readpage_limit(struct inode *inode) { if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) - return F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + return F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), + max_file_blocks(F2FS_I_SB(inode), inode)); return i_size_read(inode); } @@ -2218,7 +2452,7 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, struct readahead_control *rac) { struct bio *bio = *bio_ret; - const unsigned int blocksize = F2FS_BLKSIZE; + const unsigned int blocksize = F2FS_BLKSIZE(F2FS_I_SB(inode)); sector_t block_in_file; sector_t last_block; sector_t last_block_in_file; @@ -2228,7 +2462,8 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, block_in_file = (sector_t)index; last_block = block_in_file + nr_pages; - last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + last_block_in_file = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode), + f2fs_readpage_limit(inode) + blocksize - 1); if (last_block > last_block_in_file) last_block = last_block_in_file; @@ -2260,7 +2495,6 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, got_it: if ((map->m_flags & F2FS_MAP_MAPPED)) { block_nr = map->m_pblk + block_in_file - map->m_lblk; - folio_set_mappedtodisk(folio); if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, DATA_GENERIC_ENHANCE_READ)) { @@ -2304,9 +2538,9 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, if (!bio_add_folio(bio, folio, blocksize, 0)) goto submit_and_realloc; - inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); + inc_cache_count(F2FS_I_SB(inode), F2FS_RD_DATA); f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, - F2FS_BLKSIZE); + F2FS_BLKSIZE(F2FS_I_SB(inode))); *last_block_in_bio = block_nr; out: *bio_ret = bio; @@ -2324,7 +2558,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, struct bio *bio = *bio_ret; unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; sector_t last_block_in_file; - const unsigned int blocksize = F2FS_BLKSIZE; + const unsigned int blocksize = F2FS_BLKSIZE(sbi); struct decompress_io_ctx *dic = NULL; struct extent_info ei = {}; bool from_dnode = true; @@ -2339,7 +2573,8 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); - last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + last_block_in_file = F2FS_BYTES_TO_BLK(sbi, + f2fs_readpage_limit(inode) + blocksize - 1); /* get rid of pages beyond EOF */ @@ -2386,7 +2621,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, for (i = 1; i < cc->cluster_size; i++) { block_t blkaddr; - blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_entry, dn.ofs_in_node + i) : ei.blk + i - 1; @@ -2420,7 +2655,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, block_t blkaddr; struct bio_post_read_ctx *ctx; - blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_entry, dn.ofs_in_node + i + 1) : ei.blk + i; @@ -2455,8 +2690,9 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, ctx->enabled_steps |= STEP_DECOMPRESS; refcount_inc(&dic->refcnt); - inc_page_count(sbi, F2FS_RD_DATA); - f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); + inc_cache_count(sbi, F2FS_RD_DATA); + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, + F2FS_BLKSIZE(sbi)); *last_block_in_bio = blkaddr; } @@ -2630,14 +2866,14 @@ static int f2fs_read_data_large_folio(struct inode *inode, */ f2fs_wait_on_block_writeback(inode, block_nr); - if (!bio_add_folio(bio, folio, F2FS_BLKSIZE, - offset << PAGE_SHIFT)) + if (!bio_add_folio(bio, folio, F2FS_BLKSIZE(F2FS_I_SB(inode)), + offset << PAGE_SHIFT)) goto submit_and_realloc; folio_in_bio = true; - inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); + inc_cache_count(F2FS_I_SB(inode), F2FS_RD_DATA); f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, - F2FS_BLKSIZE); + F2FS_BLKSIZE(F2FS_I_SB(inode))); last_block_in_bio = block_nr; } trace_f2fs_read_folio(folio, DATA); @@ -2909,10 +3145,12 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) return true; if (f2fs_used_in_atomic_write(inode)) return true; - /* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */ - if (f2fs_compressed_file(inode) && - F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER && - is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) + /* + * rewrite low ratio compress data w/ OPU mode to avoid fragmentation. + * If IO comes from compressed write path and fallback to raw write, + * force out‑place to prevent metadata‑data inconsistency. + */ + if (fio && f2fs_compressed_file(inode)) return true; /* swap file is migrating in aligned write mode */ @@ -3001,7 +3239,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) goto out_writepage; } - /* wait for GCed page writeback via META_MAPPING */ + /* wait for GCed page writeback via generic cache */ if (fio->meta_gc) f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); @@ -3410,7 +3648,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, if (folio_test_writeback(folio)) { if (wbc->sync_mode == WB_SYNC_NONE) goto continue_unlock; - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); } if (!folio_clear_dirty_for_io(folio)) @@ -3492,8 +3730,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, mapping->writeback_index = done_index; if (nwritten) - f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, - NULL, 0, DATA); + f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), + mapping->host, NULL); /* submit cached bio of IPU write */ if (bio) f2fs_submit_merged_ipu_write(sbi, &bio, NULL); @@ -3549,7 +3787,7 @@ static inline void update_skipped_write(struct f2fs_sb_info *sbi, if (is_sbi_flag_set(sbi, SBI_ENABLE_CHECKPOINT) && skipped && wbc->sync_mode == WB_SYNC_ALL) - atomic_add(skipped, &sbi->nr_pages[F2FS_SKIPPED_WRITE]); + atomic_add(skipped, &sbi->nr_caches[F2FS_SKIPPED_WRITE]); } static int __f2fs_write_data_pages(struct address_space *mapping, @@ -3572,7 +3810,7 @@ static int __f2fs_write_data_pages(struct address_space *mapping, if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && wbc->sync_mode == WB_SYNC_NONE && - get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && + get_dirty_pages(inode) < nr_caches_to_skip(sbi, DATA) && f2fs_available_free_memory(sbi, DIRTY_DENTS)) goto skip_write; @@ -3671,7 +3909,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, pgoff_t index = folio->index; struct dnode_of_data dn; struct f2fs_lock_context lc; - struct folio *ifolio; + struct f2fs_cached_block *ientry; bool locked = false; int flag = F2FS_GET_BLOCK_PRE_AIO; int err = 0; @@ -3692,6 +3930,11 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, if (f2fs_has_inline_data(inode)) { if (pos + len > MAX_INLINE_DATA(inode)) flag = F2FS_GET_BLOCK_DEFAULT; + /* + * it needs to cover FI_DATA_EXIST and inline data update w/ + * node_change, otherwise concurrent checkpoint may persist + * inline data only w/o F2FS_DATA_EXIST. + */ f2fs_map_lock(sbi, &lc, flag); locked = true; } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { @@ -3701,20 +3944,20 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, restart: /* check inline_data */ - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) { - err = PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto unlock_out; } - set_new_dnode(&dn, inode, ifolio, ifolio, 0); + set_new_dnode(&dn, inode, ientry, ientry, 0); if (f2fs_has_inline_data(inode)) { if (pos + len <= MAX_INLINE_DATA(inode)) { - f2fs_do_read_inline_data(folio, ifolio); + f2fs_do_read_inline_data(folio, ientry); set_inode_flag(inode, FI_DATA_EXIST); if (inode->i_nlink) - folio_set_f2fs_inline(ifolio); + f2fs_cache_set_inline(ientry); goto out; } err = f2fs_convert_inline_folio(&dn, folio); @@ -3761,14 +4004,14 @@ static int __find_data_block(struct inode *inode, pgoff_t index, block_t *blk_addr) { struct dnode_of_data dn; - struct folio *ifolio; + struct f2fs_cached_block *ientry; int err = 0; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); - set_new_dnode(&dn, inode, ifolio, ifolio, 0); + set_new_dnode(&dn, inode, ientry, ientry, 0); if (!f2fs_lookup_read_extent_cache_block(inode, index, &dn.data_blkaddr)) { @@ -3790,17 +4033,17 @@ static int __reserve_data_block(struct inode *inode, pgoff_t index, struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; struct f2fs_lock_context lc; - struct folio *ifolio; + struct f2fs_cached_block *ientry; int err = 0; f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) { - err = PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto unlock_out; } - set_new_dnode(&dn, inode, ifolio, ifolio, 0); + set_new_dnode(&dn, inode, ientry, ientry, 0); if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, &dn.data_blkaddr)) @@ -3956,7 +4199,7 @@ static int f2fs_write_begin(const struct kiocb *iocb, } } - f2fs_folio_wait_writeback(folio, DATA, false, true); + f2fs_folio_wait_writeback(folio, false, true); if (len == folio_size(folio) || folio_test_uptodate(folio)) return 0; @@ -4073,14 +4316,8 @@ void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) return; if (folio_test_dirty(folio)) { - if (inode->i_ino == F2FS_META_INO(sbi)) { - dec_page_count(sbi, F2FS_DIRTY_META); - } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { - dec_page_count(sbi, F2FS_DIRTY_NODES); - } else { - inode_dec_dirty_pages(inode); - f2fs_remove_dirty_inode(inode); - } + inode_dec_dirty_pages(inode); + f2fs_remove_dirty_inode(inode); } if (offset || length != folio_size(folio)) @@ -4151,6 +4388,7 @@ static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) { struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); sector_t blknr = 0; if (f2fs_has_inline_data(inode)) @@ -4161,7 +4399,7 @@ static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) filemap_write_and_wait(mapping); /* Block number less than F2FS MAX BLOCKS */ - if (unlikely(block >= max_file_blocks(inode))) + if (unlikely(block >= max_file_blocks(sbi, inode))) goto out; if (f2fs_compressed_file(inode)) { @@ -4277,7 +4515,7 @@ static int check_swap_activate(struct swap_info_struct *sis, * to be very smart. */ cur_lblock = 0; - last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode)); + last_lblock = F2FS_BYTES_TO_BLK(sbi, i_size_read(inode)); while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; @@ -4363,7 +4601,8 @@ static int check_swap_activate(struct swap_info_struct *sis, out: if (not_aligned) f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)", - not_aligned, blks_per_sec * F2FS_BLKSIZE); + not_aligned, + (unsigned long)blks_per_sec * F2FS_BLKSIZE(sbi)); return ret; } @@ -4532,12 +4771,14 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_map_blocks map = { NULL, }; pgoff_t next_pgofs = 0; int err; - map.m_lblk = F2FS_BYTES_TO_BLK(offset); - map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1; + map.m_lblk = F2FS_BYTES_TO_BLK(sbi, offset); + map.m_len = F2FS_BYTES_TO_BLK(sbi, offset + length - 1) - + map.m_lblk + 1; map.m_next_pgofs = &next_pgofs; map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), inode->i_write_hint); @@ -4558,7 +4799,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (err) return err; - iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk); + iomap->offset = F2FS_BLK_TO_BYTES(sbi, map.m_lblk); /* * Sometimes I/O to an encrypted file has to be broken up to guarantee @@ -4578,11 +4819,11 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) return -EINVAL; - iomap->length = F2FS_BLK_TO_BYTES(map.m_len); + iomap->length = F2FS_BLK_TO_BYTES(sbi, map.m_len); iomap->type = IOMAP_MAPPED; iomap->flags |= IOMAP_F_MERGED; iomap->bdev = map.m_bdev; - iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk); + iomap->addr = F2FS_BLK_TO_BYTES(sbi, map.m_pblk); if (flags & IOMAP_WRITE && map.m_last_pblk) iomap->private = (void *)map.m_last_pblk; @@ -4591,11 +4832,11 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return -ENOTBLK; if (map.m_pblk == NULL_ADDR) { - iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) - + iomap->length = F2FS_BLK_TO_BYTES(sbi, next_pgofs) - iomap->offset; iomap->type = IOMAP_HOLE; } else if (map.m_pblk == NEW_ADDR) { - iomap->length = F2FS_BLK_TO_BYTES(map.m_len); + iomap->length = F2FS_BLK_TO_BYTES(sbi, map.m_len); iomap->type = IOMAP_UNWRITTEN; } else { f2fs_bug_on(F2FS_I_SB(inode), 1); diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index ff379aff4472..6fe606e2c70d 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -156,12 +156,12 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->allocated_data_blocks = atomic64_read(&sbi->allocated_data_blocks); /* validation check of the segment numbers */ - si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES); - si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); - si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); - si->ndirty_data = get_pages(sbi, F2FS_DIRTY_DATA); - si->ndirty_qdata = get_pages(sbi, F2FS_DIRTY_QDATA); - si->ndirty_imeta = get_pages(sbi, F2FS_DIRTY_IMETA); + si->ndirty_node = get_nr_caches(sbi, F2FS_DIRTY_NODES); + si->ndirty_dent = get_nr_caches(sbi, F2FS_DIRTY_DENTS); + si->ndirty_meta = get_nr_caches(sbi, F2FS_DIRTY_META); + si->ndirty_data = get_nr_caches(sbi, F2FS_DIRTY_DATA); + si->ndirty_qdata = get_nr_caches(sbi, F2FS_DIRTY_QDATA); + si->ndirty_imeta = get_nr_caches(sbi, F2FS_DIRTY_IMETA); si->ndirty_dirs = sbi->ndirty_inode[DIR_INODE]; si->ndirty_files = sbi->ndirty_inode[FILE_INODE]; si->ndonate_files = sbi->donate_files; @@ -169,13 +169,13 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->ndirty_all = sbi->ndirty_inode[DIRTY_META]; si->aw_cnt = atomic_read(&sbi->atomic_files); si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt); - si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ); - si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE); - si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA); - si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA); - si->nr_rd_data = get_pages(sbi, F2FS_RD_DATA); - si->nr_rd_node = get_pages(sbi, F2FS_RD_NODE); - si->nr_rd_meta = get_pages(sbi, F2FS_RD_META); + si->nr_dio_read = get_nr_caches(sbi, F2FS_DIO_READ); + si->nr_dio_write = get_nr_caches(sbi, F2FS_DIO_WRITE); + si->nr_wb_cp_data = get_nr_caches(sbi, F2FS_WB_CP_DATA); + si->nr_wb_data = get_nr_caches(sbi, F2FS_WB_DATA); + si->nr_rd_data = get_nr_caches(sbi, F2FS_RD_DATA); + si->nr_rd_node = get_nr_caches(sbi, F2FS_RD_NODE); + si->nr_rd_meta = get_nr_caches(sbi, F2FS_RD_META); if (SM_I(sbi)->fcc_info) { si->nr_flushed = atomic_read(&SM_I(sbi)->fcc_info->issued_flush); @@ -222,13 +222,11 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->free_secs = free_sections(sbi); si->prefree_count = prefree_segments(sbi); si->dirty_count = dirty_segments(sbi); - if (sbi->node_inode) - si->node_pages = NODE_MAPPING(sbi)->nrpages; - if (sbi->meta_inode) - si->meta_pages = META_MAPPING(sbi)->nrpages; + si->node_caches = NODE_CACHE(sbi)->num_entries; + si->meta_caches = META_CACHE(sbi)->num_entries; #ifdef CONFIG_F2FS_FS_COMPRESSION - if (sbi->compress_inode) { - si->compress_pages = COMPRESS_MAPPING(sbi)->nrpages; + if (test_opt(sbi, COMPRESS_CACHE)) { + si->compress_pages = COMPRESS_CACHE(sbi)->num_entries; si->compress_page_hit = atomic_read(&sbi->compress_page_hit); } #endif @@ -343,9 +341,9 @@ static void update_mem_info(struct f2fs_sb_info *sbi) /* build nm */ si->base_mem += sizeof(struct f2fs_nm_info); si->base_mem += __bitmap_size(sbi, NAT_BITMAP); - si->base_mem += F2FS_BLK_TO_BYTES(NM_I(sbi)->nat_bits_blocks); + si->base_mem += F2FS_BLK_TO_BYTES(sbi, NM_I(sbi)->nat_bits_blocks); si->base_mem += NM_I(sbi)->nat_blocks * - f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK); + f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK(sbi)); si->base_mem += NM_I(sbi)->nat_blocks / 8; si->base_mem += NM_I(sbi)->nat_blocks * sizeof(unsigned short); @@ -382,22 +380,36 @@ static void update_mem_info(struct f2fs_sb_info *sbi) si->cache_mem += si->ext_mem[i]; } - si->page_mem = 0; - if (sbi->node_inode) { - unsigned long npages = NODE_MAPPING(sbi)->nrpages; + si->cache_entry_mem[F2FS_META_CACHE] = + (unsigned long long)META_CACHE(sbi)->num_entries * + sizeof(struct f2fs_cached_block); + si->cache_data_mem[F2FS_META_CACHE] = + (unsigned long long)META_CACHE(sbi)->num_entries * sbi->blocksize; - si->page_mem += (unsigned long long)npages << PAGE_SHIFT; - } - if (sbi->meta_inode) { - unsigned long npages = META_MAPPING(sbi)->nrpages; + si->cache_entry_mem[F2FS_NODE_CACHE] = + (unsigned long long)NODE_CACHE(sbi)->num_entries * + sizeof(struct f2fs_cached_block); + si->cache_data_mem[F2FS_NODE_CACHE] = + (unsigned long long)NODE_CACHE(sbi)->num_entries * sbi->blocksize; - si->page_mem += (unsigned long long)npages << PAGE_SHIFT; - } + si->cache_mem += si->cache_entry_mem[F2FS_META_CACHE] + + si->cache_entry_mem[F2FS_NODE_CACHE]; + si->page_mem = si->cache_data_mem[F2FS_META_CACHE] + + si->cache_data_mem[F2FS_NODE_CACHE]; #ifdef CONFIG_F2FS_FS_COMPRESSION - if (sbi->compress_inode) { - unsigned long npages = COMPRESS_MAPPING(sbi)->nrpages; + if (test_opt(sbi, COMPRESS_CACHE)) { + si->cache_entry_mem[F2FS_COMPRESS_CACHE] = + (unsigned long long)COMPRESS_CACHE(sbi)->num_entries * + sizeof(struct f2fs_cached_block); + si->cache_data_mem[F2FS_COMPRESS_CACHE] = + (unsigned long long)COMPRESS_CACHE(sbi)->num_entries * + sbi->blocksize; - si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + si->cache_mem += si->cache_entry_mem[F2FS_COMPRESS_CACHE]; + si->page_mem += si->cache_data_mem[F2FS_COMPRESS_CACHE]; + } else { + si->cache_entry_mem[F2FS_COMPRESS_CACHE] = 0; + si->cache_data_mem[F2FS_COMPRESS_CACHE] = 0; } #endif } @@ -700,7 +712,7 @@ static int stat_show(struct seq_file *s, void *v) si->aw_cnt, si->max_aw_cnt); seq_printf(s, " - compress: %4d, hit:%8d\n", si->compress_pages, si->compress_page_hit); seq_printf(s, " - nodes: %4d in %4d\n", - si->ndirty_node, si->node_pages); + si->ndirty_node, si->node_caches); seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); seq_printf(s, " - data: %4d in files:%4d\n", @@ -708,7 +720,7 @@ static int stat_show(struct seq_file *s, void *v) seq_printf(s, " - quota data: %4d in quota files:%4d\n", si->ndirty_qdata, si->nquota_files); seq_printf(s, " - meta: %4d in %4d\n", - si->ndirty_meta, si->meta_pages); + si->ndirty_meta, si->meta_caches); seq_printf(s, " - imeta: %4d\n", si->ndirty_imeta); seq_printf(s, " - fsync mark: %4lld\n", @@ -756,6 +768,18 @@ static int stat_show(struct seq_file *s, void *v) si->ext_mem[EX_READ] >> 10); seq_printf(s, " - block age extent cache: %llu KB\n", si->ext_mem[EX_BLOCK_AGE] >> 10); + seq_printf(s, " - meta entry: %llu KB, meta cache: %llu KB\n", + si->cache_entry_mem[F2FS_META_CACHE] >> 10, + si->cache_data_mem[F2FS_META_CACHE] >> 10); + seq_printf(s, " - node entry: %llu KB, node cache: %llu KB\n", + si->cache_entry_mem[F2FS_NODE_CACHE] >> 10, + si->cache_data_mem[F2FS_NODE_CACHE] >> 10); +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (test_opt(sbi, COMPRESS_CACHE)) + seq_printf(s, " - compress entry: %llu KB, compress cache: %llu KB\n", + si->cache_entry_mem[F2FS_COMPRESS_CACHE] >> 10, + si->cache_data_mem[F2FS_COMPRESS_CACHE] >> 10); +#endif seq_printf(s, " - paged : %llu KB\n", si->page_mem >> 10); } diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index fd0e2cd31a81..97f7fef6b068 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -195,7 +195,7 @@ static struct f2fs_dir_entry *find_in_block(struct inode *dir, int *max_slots, bool use_hash) { - struct f2fs_dentry_block *dentry_blk; + void *dentry_blk; struct f2fs_dentry_ptr d; dentry_blk = folio_address(dentry_folio); @@ -282,7 +282,7 @@ struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d, static struct f2fs_dir_entry *find_in_level(struct inode *dir, unsigned int level, const struct f2fs_filename *fname, - struct folio **res_folio, + void **dentry_block, bool use_hash) { int s = GET_DENTRY_SLOTS(fname->disk_name.len); @@ -313,7 +313,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir, bidx = next_pgofs; continue; } else { - *res_folio = dentry_folio; + *dentry_block = dentry_folio; break; } } @@ -321,11 +321,11 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir, de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash); if (IS_ERR(de)) { f2fs_folio_put(dentry_folio, false); - *res_folio = ERR_CAST(de); + *dentry_block = ERR_CAST(de); de = NULL; break; } else if (de) { - *res_folio = dentry_folio; + *dentry_block = dentry_folio; break; } @@ -352,7 +352,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir, struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, const struct f2fs_filename *fname, - struct folio **res_folio) + void **dentry_block) { unsigned long npages = dir_blocks(dir); struct f2fs_dir_entry *de = NULL; @@ -360,13 +360,13 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, unsigned int level; bool use_hash = true; - *res_folio = NULL; + *dentry_block = NULL; #if IS_ENABLED(CONFIG_UNICODE) start_find_entry: #endif if (f2fs_has_inline_dentry(dir)) { - de = f2fs_find_in_inline_dir(dir, fname, res_folio, use_hash); + de = f2fs_find_in_inline_dir(dir, fname, dentry_block, use_hash); goto out; } @@ -382,8 +382,8 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, } for (level = 0; level < max_depth; level++) { - de = find_in_level(dir, level, fname, res_folio, use_hash); - if (de || IS_ERR(*res_folio)) + de = find_in_level(dir, level, fname, dentry_block, use_hash); + if (de || IS_ERR(*dentry_block)) break; } @@ -408,7 +408,7 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, * Entry is guaranteed to be valid. */ struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, - const struct qstr *child, struct folio **res_folio) + const struct qstr *child, void **dentry_block) { struct f2fs_dir_entry *de = NULL; struct f2fs_filename fname; @@ -417,67 +417,78 @@ struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, err = f2fs_setup_filename(dir, child, 1, &fname); if (err) { if (err == -ENOENT) - *res_folio = NULL; + *dentry_block = NULL; else - *res_folio = ERR_PTR(err); + *dentry_block = ERR_PTR(err); return NULL; } - de = __f2fs_find_entry(dir, &fname, res_folio); + de = __f2fs_find_entry(dir, &fname, dentry_block); f2fs_free_filename(&fname); return de; } -struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f) +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, void **dentry_block) { - return f2fs_find_entry(dir, &dotdot_name, f); + return f2fs_find_entry(dir, &dotdot_name, dentry_block); } ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr, - struct folio **folio) + void **dentry_block) { ino_t res = 0; struct f2fs_dir_entry *de; - de = f2fs_find_entry(dir, qstr, folio); + de = f2fs_find_entry(dir, qstr, dentry_block); if (de) { res = le32_to_cpu(de->ino); - f2fs_folio_put(*folio, false); + f2fs_put_dentry_block(*dentry_block, false); } return res; } void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, - struct folio *folio, struct inode *inode) + void *dentry_block, struct inode *inode) { - enum page_type type = f2fs_has_inline_dentry(dir) ? NODE : DATA; + if (f2fs_dentry_is_cache(dentry_block)) { + struct f2fs_cached_block *entry = + f2fs_dentry_cache(dentry_block); - folio_lock(folio); - f2fs_folio_wait_writeback(folio, type, true, true); - de->ino = cpu_to_le32(inode->i_ino); - de->file_type = fs_umode_to_ftype(inode->i_mode); - folio_mark_dirty(folio); + f2fs_lock_cache(entry); + f2fs_cache_wait_writeback(entry); + de->ino = cpu_to_le32(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + f2fs_mark_cache_dirty(entry); + } else { + struct folio *folio = f2fs_dentry_folio(dentry_block); + + folio_lock(folio); + f2fs_folio_wait_writeback(folio, true, true); + de->ino = cpu_to_le32(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + folio_mark_dirty(folio); + } inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); f2fs_mark_inode_dirty_sync(dir, true); - f2fs_folio_put(folio, true); + f2fs_put_dentry_block(dentry_block, true); } static void init_dent_inode(struct inode *dir, struct inode *inode, const struct f2fs_filename *fname, - struct folio *ifolio) + struct f2fs_cached_block *ientry) { struct f2fs_inode *ri; if (!fname) /* tmpfile case? */ return; - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); /* copy name info. to this inode folio */ - ri = F2FS_INODE(ifolio); + ri = F2FS_INODE(ientry); ri->i_namelen = cpu_to_le32(fname->disk_name.len); memcpy(ri->i_name, fname->disk_name.name, fname->disk_name.len); if (IS_ENCRYPTED(dir)) { @@ -498,7 +509,7 @@ static void init_dent_inode(struct inode *dir, struct inode *inode, file_lost_pino(inode); } } - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); } void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent, @@ -515,22 +526,22 @@ void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent, } static int make_empty_dir(struct inode *inode, - struct inode *parent, struct folio *folio) + struct inode *parent, struct f2fs_cached_block *ientry) { struct folio *dentry_folio; - struct f2fs_dentry_block *dentry_blk; + void *dentry_blk; struct f2fs_dentry_ptr d; if (f2fs_has_inline_dentry(inode)) - return f2fs_make_empty_inline_dir(inode, parent, folio); + return f2fs_make_empty_inline_dir(inode, parent, ientry); - dentry_folio = f2fs_get_new_data_folio(inode, folio, 0, true); + dentry_folio = f2fs_get_new_data_folio(inode, ientry, 0, true); if (IS_ERR(dentry_folio)) return PTR_ERR(dentry_folio); dentry_blk = folio_address(dentry_folio); - make_dentry_ptr_block(NULL, &d, dentry_blk); + make_dentry_ptr_block(inode, &d, dentry_blk); f2fs_do_make_empty_dir(inode, parent, &d); folio_mark_dirty(dentry_folio); @@ -538,50 +549,50 @@ static int make_empty_dir(struct inode *inode, return 0; } -struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, - const struct f2fs_filename *fname, struct folio *dfolio) +struct f2fs_cached_block *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, + const struct f2fs_filename *fname, struct f2fs_cached_block *dentry) { - struct folio *folio; + struct f2fs_cached_block *ientry; int err; if (is_inode_flag_set(inode, FI_NEW_INODE)) { - folio = f2fs_new_inode_folio(inode); - if (IS_ERR(folio)) - return folio; + ientry = f2fs_new_inode_cache(inode); + if (IS_ERR(ientry)) + return ientry; if (S_ISDIR(inode->i_mode)) { /* in order to handle error case */ - folio_get(folio); - err = make_empty_dir(inode, dir, folio); + f2fs_cache_get(ientry); + err = make_empty_dir(inode, dir, ientry); if (err) { - folio_lock(folio); + f2fs_lock_cache(ientry); goto put_error; } - folio_put(folio); + f2fs_put_cache(ientry, false); } - err = f2fs_init_acl(inode, dir, folio, dfolio); + err = f2fs_init_acl(inode, dir, ientry, dentry); if (err) goto put_error; err = f2fs_init_security(inode, dir, fname ? fname->usr_fname : NULL, - folio); + ientry); if (err) goto put_error; if (IS_ENCRYPTED(inode)) { - err = fscrypt_set_context(inode, folio); + err = fscrypt_set_context(inode, ientry); if (err) goto put_error; } } else { - folio = f2fs_get_inode_folio(F2FS_I_SB(dir), inode->i_ino); - if (IS_ERR(folio)) - return folio; + ientry = f2fs_get_inode_cache(F2FS_I_SB(dir), inode->i_ino); + if (IS_ERR(ientry)) + return ientry; } - init_dent_inode(dir, inode, fname, folio); + init_dent_inode(dir, inode, fname, ientry); /* * This file should be checkpointed during fsync. @@ -598,12 +609,12 @@ struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, f2fs_remove_orphan_inode(F2FS_I_SB(dir), inode->i_ino); f2fs_i_links_write(inode, true); } - return folio; + return ientry; put_error: clear_nlink(inode); - f2fs_update_inode(inode, folio); - f2fs_folio_put(folio, true); + f2fs_update_inode(inode, ientry); + f2fs_put_cache(ientry, true); return ERR_PTR(err); } @@ -645,14 +656,14 @@ int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots) goto next; } -bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio, +bool f2fs_has_enough_room(struct inode *dir, struct f2fs_cached_block *ientry, const struct f2fs_filename *fname) { struct f2fs_dentry_ptr d; unsigned int bit_pos; int slots = GET_DENTRY_SLOTS(fname->disk_name.len); - make_dentry_ptr_inline(dir, &d, inline_data_addr(dir, ifolio)); + make_dentry_ptr_inline(dir, &d, inline_data_addr(dir, ientry)); bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max); @@ -690,9 +701,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, unsigned long bidx, block; unsigned int nbucket, nblock; struct folio *dentry_folio = NULL; - struct f2fs_dentry_block *dentry_blk = NULL; + void *dentry_blk = NULL; struct f2fs_dentry_ptr d; - struct folio *folio = NULL; + struct f2fs_cached_block *entry = NULL; int slots, err = 0; level = 0; @@ -727,9 +738,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, return PTR_ERR(dentry_folio); dentry_blk = folio_address(dentry_folio); - bit_pos = f2fs_room_for_filename(&dentry_blk->dentry_bitmap, - slots, NR_DENTRY_IN_BLOCK); - if (bit_pos < NR_DENTRY_IN_BLOCK) + make_dentry_ptr_block(dir, &d, dentry_blk); + bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max); + if (bit_pos < d.max) goto add_dentry; f2fs_folio_put(dentry_folio, true); @@ -739,18 +750,17 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, ++level; goto start; add_dentry: - f2fs_folio_wait_writeback(dentry_folio, DATA, true, true); + f2fs_folio_wait_writeback(dentry_folio, true, true); if (inode) { f2fs_down_write(&F2FS_I(inode)->i_sem); - folio = f2fs_init_inode_metadata(inode, dir, fname, NULL); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + entry = f2fs_init_inode_metadata(inode, dir, fname, NULL); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); goto fail; } } - make_dentry_ptr_block(NULL, &d, dentry_blk); f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash, bit_pos); @@ -761,9 +771,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, /* synchronize inode page's data from inode cache */ if (is_inode_flag_set(inode, FI_NEW_INODE)) - f2fs_update_inode(inode, folio); + f2fs_update_inode(inode, entry); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } f2fs_update_parent_metadata(dir, inode, current_depth); @@ -805,7 +815,7 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name, struct inode *inode, nid_t ino, umode_t mode) { struct f2fs_filename fname; - struct folio *folio = NULL; + void *dentry_blk = NULL; struct f2fs_dir_entry *de = NULL; int err; @@ -821,14 +831,14 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name, * consistency more. */ if (current != F2FS_I(dir)->task) { - de = __f2fs_find_entry(dir, &fname, &folio); + de = __f2fs_find_entry(dir, &fname, &dentry_blk); F2FS_I(dir)->task = NULL; } if (de) { - f2fs_folio_put(folio, false); + f2fs_put_dentry_block(dentry_blk, false); err = -EEXIST; - } else if (IS_ERR(folio)) { - err = PTR_ERR(folio); + } else if (IS_ERR(dentry_blk)) { + err = PTR_ERR(dentry_blk); } else { err = f2fs_add_dentry(dir, &fname, inode, ino, mode); } @@ -839,16 +849,16 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name, int f2fs_do_tmpfile(struct inode *inode, struct inode *dir, struct f2fs_filename *fname) { - struct folio *folio; + struct f2fs_cached_block *ientry; int err = 0; f2fs_down_write(&F2FS_I(inode)->i_sem); - folio = f2fs_init_inode_metadata(inode, dir, fname, NULL); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + ientry = f2fs_init_inode_metadata(inode, dir, fname, NULL); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto fail; } - f2fs_folio_put(folio, true); + f2fs_put_cache(ientry, true); clear_inode_flag(inode, FI_NEW_INODE); f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); @@ -884,13 +894,15 @@ void f2fs_drop_nlink(struct inode *dir, struct inode *inode) * It only removes the dentry from the dentry page, corresponding name * entry in name page does not need to be touched during deletion. */ -void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, +void f2fs_delete_entry(struct f2fs_dir_entry *dentry, void *dentry_block, struct inode *dir, struct inode *inode) { - struct f2fs_dentry_block *dentry_blk; + void *dentry_blk; + struct f2fs_dentry_ptr d; + struct folio *folio; unsigned int bit_pos; int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); - pgoff_t index = folio->index; + pgoff_t index; int i; f2fs_update_time(F2FS_I_SB(dir), REQ_TIME); @@ -899,24 +911,27 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, f2fs_add_ino_entry(F2FS_I_SB(dir), dir->i_ino, TRANS_DIR_INO); if (f2fs_has_inline_dentry(dir)) - return f2fs_delete_inline_entry(dentry, folio, dir, inode); + return f2fs_delete_inline_entry(dentry, + f2fs_dentry_cache(dentry_block), dir, inode); + + folio = f2fs_dentry_folio(dentry_block); + index = folio->index; folio_lock(folio); - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); dentry_blk = folio_address(folio); - bit_pos = dentry - dentry_blk->dentry; + make_dentry_ptr_block(dir, &d, dentry_blk); + bit_pos = dentry - d.dentry; for (i = 0; i < slots; i++) - __clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap); + __clear_bit_le(bit_pos + i, d.bitmap); /* Let's check and deallocate this dentry page */ - bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, - NR_DENTRY_IN_BLOCK, - 0); + bit_pos = find_next_bit_le(d.bitmap, d.max, 0); folio_mark_dirty(folio); - if (bit_pos == NR_DENTRY_IN_BLOCK && - !f2fs_truncate_hole(dir, index, index + 1)) { + if (bit_pos == d.max && + !f2fs_truncate_hole(dir, index, index + 1)) { f2fs_clear_page_cache_dirty_tag(folio); folio_clear_dirty_for_io(folio); folio_clear_uptodate(folio); @@ -938,7 +953,8 @@ bool f2fs_empty_dir(struct inode *dir) { unsigned long bidx = 0; unsigned int bit_pos; - struct f2fs_dentry_block *dentry_blk; + void *dentry_blk; + struct f2fs_dentry_ptr d; unsigned long nblock = dir_blocks(dir); if (f2fs_has_inline_dentry(dir)) @@ -959,17 +975,16 @@ bool f2fs_empty_dir(struct inode *dir) } dentry_blk = folio_address(dentry_folio); + make_dentry_ptr_block(dir, &d, dentry_blk); if (bidx == 0) bit_pos = 2; else bit_pos = 0; - bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, - NR_DENTRY_IN_BLOCK, - bit_pos); + bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos); f2fs_folio_put(dentry_folio, false); - if (bit_pos < NR_DENTRY_IN_BLOCK) + if (bit_pos < d.max) return false; bidx++; @@ -1051,7 +1066,7 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, } if (readdir_ra) - f2fs_ra_node_page(sbi, le32_to_cpu(de->ino)); + f2fs_ra_node_cache(sbi, le32_to_cpu(de->ino)); ctx->pos = start_pos + bit_pos; found_valid_dirent = true; @@ -1066,10 +1081,12 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) { struct inode *inode = file_inode(file); unsigned long npages = dir_blocks(inode); - struct f2fs_dentry_block *dentry_blk = NULL; + void *dentry_blk = NULL; struct file_ra_state *ra = &file->f_ra; loff_t start_pos = ctx->pos; - unsigned int n = ((unsigned long)ctx->pos / NR_DENTRY_IN_BLOCK); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int entries = sbi->dentries_per_block; + unsigned int n = (unsigned long)ctx->pos / entries; struct f2fs_dentry_ptr d; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); int err = 0; @@ -1089,7 +1106,7 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) goto out_free; } - for (; n < npages; ctx->pos = n * NR_DENTRY_IN_BLOCK) { + for (; n < npages; ctx->pos = n * entries) { struct folio *dentry_folio; pgoff_t next_pgofs; @@ -1122,7 +1139,7 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) make_dentry_ptr_block(inode, &d, dentry_blk); err = f2fs_fill_dentries(ctx, &d, - n * NR_DENTRY_IN_BLOCK, &fstr); + n * entries, &fstr); f2fs_folio_put(dentry_folio, false); if (err) break; diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 37cf9fa8d537..5a944428aca0 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -20,10 +20,10 @@ #include "segment.h" #include -bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio) +bool sanity_check_extent_cache(struct inode *inode, struct f2fs_cached_block *ientry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext; + struct f2fs_extent *i_ext = &F2FS_INODE(ientry)->i_ext; struct extent_info ei; int devi; @@ -416,11 +416,11 @@ static void __drop_largest_extent(struct extent_tree *et, } } -void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio) +void f2fs_init_read_extent_tree(struct inode *inode, struct f2fs_cached_block *ientry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct extent_tree_info *eti = &sbi->extent_tree[EX_READ]; - struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext; + struct f2fs_extent *i_ext = &F2FS_INODE(ientry)->i_ext; struct extent_tree *et; struct extent_node *en; struct extent_info ei = {0}; @@ -428,9 +428,9 @@ void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio) if (!__may_extent_tree(inode, EX_READ)) { /* drop largest read extent */ if (i_ext->len) { - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); i_ext->len = 0; - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); } set_inode_flag(inode, FI_NO_EXTENT); return; @@ -904,11 +904,12 @@ static int __get_new_block_age(struct inode *inode, struct extent_info *ei, struct extent_info tei = *ei; /* only fofs and len are valid */ /* - * When I/O is not aligned to a PAGE_SIZE, update will happen to the last - * file block even in seq write. So don't record age for newly last file - * block here. + * When I/O is not aligned to the filesystem block size, update will + * happen to the last file block even in seq write. Do not record the age + * of the new last file block here. */ - if ((f_size >> PAGE_SHIFT) == ei->fofs && f_size & (PAGE_SIZE - 1) && + if ((f_size >> sbi->log_blocksize) == ei->fofs && + f_size & F2FS_BLKSIZE_MASK(sbi) && blkaddr == NEW_ADDR) return -EINVAL; @@ -956,8 +957,8 @@ static void __update_extent_cache(struct dnode_of_data *dn, enum extent_type typ if (!__may_extent_tree(dn->inode, type)) return; - ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), dn->inode) + - dn->ofs_in_node; + ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(F2FS_I_SB(dn->inode), + dn->node_entry), dn->inode) + dn->ofs_in_node; ei.len = 1; if (type == EX_READ) { diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index e3108c5f7c91..f7423b3f6e4a 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -149,11 +149,11 @@ enum f2fs_mount_opt { #define F2FS_OPTION(sbi) ((sbi)->mount_opt) #define clear_opt(sbi, option) \ - (F2FS_OPTION(sbi).opt &= ~BIT(F2FS_MOUNT_##option)) + (F2FS_OPTION(sbi).opt &= ~BIT_ULL(F2FS_MOUNT_##option)) #define set_opt(sbi, option) \ - (F2FS_OPTION(sbi).opt |= BIT(F2FS_MOUNT_##option)) + (F2FS_OPTION(sbi).opt |= BIT_ULL(F2FS_MOUNT_##option)) #define test_opt(sbi, option) \ - (F2FS_OPTION(sbi).opt & BIT(F2FS_MOUNT_##option)) + (F2FS_OPTION(sbi).opt & BIT_ULL(F2FS_MOUNT_##option)) #define ver_after(a, b) (typecheck(unsigned long long, a) && \ typecheck(unsigned long long, b) && \ @@ -222,6 +222,8 @@ struct f2fs_rwsem { #endif }; +#include "cache.h" + struct f2fs_mount_info { unsigned long long opt; block_t root_reserved_blocks; /* root reserved blocks */ @@ -416,7 +418,7 @@ struct inode_entry { struct fsync_node_entry { struct list_head list; /* list head */ - struct folio *folio; /* warm node folio pointer */ + struct f2fs_cached_block *entry; /* warm node cache entry pointer */ unsigned int seq_id; /* sequence id */ }; @@ -466,6 +468,7 @@ struct discard_entry { #define MAX_PLIST_NUM 512 #define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \ (MAX_PLIST_NUM - 1) : ((blk_num) - 1)) +#define MAX_DISCARD_DROP_COUNT 512 enum { D_PREP, /* initial */ @@ -602,8 +605,9 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) #define DEF_INLINE_RESERVED_SIZE 1 static inline int get_extra_isize(struct inode *inode); static inline int get_inline_xattr_addrs(struct inode *inode); +static inline unsigned int cur_addrs_per_inode(struct inode *inode); #define MAX_INLINE_DATA(inode) (sizeof(__le32) * \ - (CUR_ADDRS_PER_INODE(inode) - \ + (cur_addrs_per_inode(inode) - \ get_inline_xattr_addrs(inode) - \ DEF_INLINE_RESERVED_SIZE)) @@ -669,17 +673,6 @@ struct f2fs_dentry_ptr { int nr_bitmap; }; -static inline void make_dentry_ptr_block(struct inode *inode, - struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t) -{ - d->inode = inode; - d->max = NR_DENTRY_IN_BLOCK; - d->nr_bitmap = SIZE_OF_DENTRY_BITMAP; - d->bitmap = t->dentry_bitmap; - d->dentry = t->dentry; - d->filename = t->filename; -} - static inline void make_dentry_ptr_inline(struct inode *inode, struct f2fs_dentry_ptr *d, void *t) { @@ -1081,7 +1074,7 @@ struct f2fs_nm_info { nid_t next_scan_nid; /* the next nid to be scanned */ nid_t max_rf_node_blocks; /* max # of nodes for recovery */ unsigned int ram_thresh; /* control the memory footprint */ - unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */ + unsigned int ra_nid_blocks; /* # of nid blocks to be readaheaded */ unsigned int dirty_nats_ratio; /* control dirty nats ratio threshold */ /* NAT cache management */ @@ -1123,11 +1116,11 @@ struct f2fs_nm_info { */ struct dnode_of_data { struct inode *inode; /* vfs inode pointer */ - struct folio *inode_folio; /* its inode folio, NULL is possible */ - struct folio *node_folio; /* cached direct node folio */ + struct f2fs_cached_block *inode_entry; /* generic cache inode entry */ + struct f2fs_cached_block *node_entry; /* generic cache node entry */ nid_t nid; /* node id of the direct node block */ unsigned int ofs_in_node; /* data offset in the node page */ - bool inode_folio_locked; /* inode folio is locked or not */ + bool inode_entry_locked; /* inode entry is locked or not */ bool node_changed; /* is node block changed */ char cur_level; /* level of hole node page */ char max_level; /* level of current page located */ @@ -1135,12 +1128,12 @@ struct dnode_of_data { }; static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode, - struct folio *ifolio, struct folio *nfolio, nid_t nid) + struct f2fs_cached_block *ientry, struct f2fs_cached_block *nentry, nid_t nid) { memset(dn, 0, sizeof(*dn)); dn->inode = inode; - dn->inode_folio = ifolio; - dn->node_folio = nfolio; + dn->inode_entry = ientry; + dn->node_entry = nentry; dn->nid = nid; } @@ -1368,8 +1361,10 @@ struct f2fs_io_info { unsigned int in_list:1; /* indicate fio is in io_list */ unsigned int is_por:1; /* indicate IO is from recovery or not */ unsigned int meta_gc:1; /* require meta inode GC */ + unsigned int is_cache:1; /* indicate IO is from internal cache */ enum iostat_type io_type; /* io type */ struct writeback_control *io_wbc; /* writeback control */ + struct f2fs_cached_block *cache_entry; struct bio **bio; /* bio for ipu */ sector_t *last_block; /* last block number in bio */ }; @@ -1616,10 +1611,9 @@ static inline void f2fs_clear_bit(unsigned int nr, char *addr); * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | * bit 0 F2FS_FOLIO_PRIVATE_NOT_POINTER * bit 1 F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION - * bit 2 F2FS_FOLIO_PRIVATE_INLINE_INODE - * bit 3 F2FS_FOLIO_PRIVATE_REF_RESOURCE - * bit 4 F2FS_FOLIO_PRIVATE_ATOMIC_WRITE - * bit 5- f2fs private data + * bit 2 F2FS_FOLIO_PRIVATE_REF_RESOURCE + * bit 3 F2FS_FOLIO_PRIVATE_ATOMIC_WRITE + * bit 4- f2fs private data * * Layout B: lowest bit should be 0 * folio->private is a wrapped pointer. @@ -1627,7 +1621,6 @@ static inline void f2fs_clear_bit(unsigned int nr, char *addr); enum { F2FS_FOLIO_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ - F2FS_FOLIO_PRIVATE_INLINE_INODE, /* inode page contains inline data */ F2FS_FOLIO_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ F2FS_FOLIO_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ F2FS_FOLIO_PRIVATE_MAX @@ -1779,6 +1772,13 @@ struct f2fs_gc_kthread { unsigned int boost_gc_greedy; }; +struct f2fs_bio { + struct work_struct work; + struct f2fs_cached_block *entry; + unsigned int entry_cnt; + struct bio bio; +}; + struct f2fs_sb_info { struct super_block *sb; /* pointer to VFS super block */ struct proc_dir_entry *s_proc; /* proc entry */ @@ -1798,7 +1798,6 @@ struct f2fs_sb_info { /* for node-related operations */ struct f2fs_nm_info *nm_info; /* node manager */ - struct inode *node_inode; /* cache node blocks */ /* for segment-related operations */ struct f2fs_sm_info *sm_info; /* segment manager */ @@ -1806,6 +1805,8 @@ struct f2fs_sb_info { /* for bio operations */ /* Largest write bio size completed in atomic context (atc). */ u32 max_atc_write_bio_size; + /* Largest write bio entry count completed in atomic context (atc). */ + u32 max_atc_write_bio_entry_cnt; struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */ /* keep migration IO order for LFS mode */ struct f2fs_rwsem io_order_lock; @@ -1816,7 +1817,6 @@ struct f2fs_sb_info { struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ int cur_cp_pack; /* remain current cp pack */ spinlock_t cp_lock; /* for flag in ckpt */ - struct inode *meta_inode; /* cache meta blocks */ struct f2fs_rwsem cp_global_sem; /* checkpoint procedure lock */ struct f2fs_rwsem cp_rwsem; /* blocking FS operations */ struct f2fs_rwsem node_write; /* locking node writes */ @@ -1859,9 +1859,17 @@ struct f2fs_sb_info { unsigned int log_sectors_per_block; /* log2 sectors per block */ unsigned int log_blocksize; /* log2 block size */ unsigned int blocksize; /* block size */ + unsigned int nat_entries_per_block; /* NAT entries in a block */ + unsigned int addrs_per_inode; /* addresses in an inode block */ + unsigned int addrs_per_block; /* addresses in a direct node block */ + unsigned int nids_per_block; /* node IDs in an indirect node block */ + unsigned int sit_entries_per_block; /* SIT entries in a block */ + unsigned int orphans_per_block; /* orphan inodes in a block */ + unsigned int dentries_per_block; /* dentries in a block */ + unsigned int dentry_bitmap_size; /* dentry bitmap size in bytes */ + unsigned int dentry_reserved_size; /* dentry reserved bytes */ unsigned int root_ino_num; /* root inode number*/ unsigned int node_ino_num; /* node inode number*/ - unsigned int meta_ino_num; /* meta inode number*/ unsigned int log_blocks_per_seg; /* log2 blocks per segment */ unsigned int blocks_per_seg; /* blocks per segment */ unsigned int segs_per_sec; /* segments per section */ @@ -1897,8 +1905,8 @@ struct f2fs_sb_info { struct f2fs_rwsem quota_sem; /* blocking cp for flags */ struct task_struct *umount_lock_holder; /* s_umount lock holder */ - /* # of pages, see count_type */ - atomic_t nr_pages[NR_COUNT_TYPE]; + /* # of cache entries, see count_type */ + atomic_t nr_caches[NR_COUNT_TYPE]; /* # of allocated blocks */ struct percpu_counter alloc_valid_block_count; /* # of node block writes as roll forward recovery */ @@ -2069,7 +2077,6 @@ struct f2fs_sb_info { u32 compr_new_inode; /* For compressed block cache */ - struct inode *compress_inode; /* cache compressed blocks */ unsigned int compress_percent; /* cache page percentage */ unsigned int compress_watermark; /* cache page watermark */ atomic_t compress_page_hit; /* cache hit count */ @@ -2094,6 +2101,14 @@ struct f2fs_sb_info { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lock_class_key cp_global_sem_key; #endif + + /* f2fs internal cache */ + struct f2fs_cached_block_list meta_blocks; + struct f2fs_cached_block_list node_blocks; + struct f2fs_cached_block_list compress_blocks; + + /* internal cache flush thread */ + struct f2fs_cache_kthread cache_thread; }; /* Definitions to access f2fs_sb_info */ @@ -2247,6 +2262,47 @@ static inline struct f2fs_sb_info *F2FS_F_SB(const struct folio *folio) return F2FS_M_SB(folio->mapping); } +#define SIT_ENTRY_PER_BLOCK(sbi) ((sbi)->sit_entries_per_block) +#define NAT_ENTRY_PER_BLOCK(sbi) ((sbi)->nat_entries_per_block) +#define DEF_ADDRS_PER_INODE(sbi) ((sbi)->addrs_per_inode) +#define DEF_ADDRS_PER_BLOCK(sbi) ((sbi)->addrs_per_block) +#define NIDS_PER_BLOCK(sbi) ((sbi)->nids_per_block) +#define F2FS_ORPHANS_PER_BLOCK(sbi) ((sbi)->orphans_per_block) +#define GET_ORPHAN_BLOCKS(sbi, n) DIV_ROUND_UP((n), \ + F2FS_ORPHANS_PER_BLOCK(sbi)) +#define CP_CHKSUM_OFFSET(sbi) (F2FS_BLKSIZE(sbi) - sizeof(__le32)) + +#define NODE_DIR1_BLOCK(sbi) (DEF_ADDRS_PER_INODE(sbi) + 1) +#define NODE_DIR2_BLOCK(sbi) (DEF_ADDRS_PER_INODE(sbi) + 2) +#define NODE_IND1_BLOCK(sbi) (DEF_ADDRS_PER_INODE(sbi) + 3) +#define NODE_IND2_BLOCK(sbi) (DEF_ADDRS_PER_INODE(sbi) + 4) +#define NODE_DIND_BLOCK(sbi) (DEF_ADDRS_PER_INODE(sbi) + 5) + +static inline struct f2fs_orphan_footer * +f2fs_orphan_footer(void *orphan_block, struct f2fs_sb_info *sbi) +{ + return (struct f2fs_orphan_footer *) + ((char *)orphan_block + sbi->blocksize - + sizeof(struct f2fs_orphan_footer)); +} + +static inline void make_dentry_ptr_block(struct inode *inode, + struct f2fs_dentry_ptr *d, void *t) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int entries = sbi->dentries_per_block; + unsigned int bitmap_size = sbi->dentry_bitmap_size; + unsigned int reserved_size = sbi->dentry_reserved_size; + + d->inode = inode; + d->max = entries; + d->nr_bitmap = bitmap_size; + d->bitmap = t; + d->dentry = t + bitmap_size + reserved_size; + d->filename = t + bitmap_size + reserved_size + + SIZE_OF_DIR_ENTRY * entries; +} + static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi) { return (struct f2fs_super_block *)(sbi->raw_super); @@ -2267,14 +2323,30 @@ static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi) return (struct f2fs_checkpoint *)(sbi->ckpt); } -static inline struct f2fs_node *F2FS_NODE(const struct folio *folio) +static inline struct node_footer *F2FS_NODE_FOOTER(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - return (struct f2fs_node *)folio_address(folio); + return (struct node_footer *)(cache_address(entry) + + F2FS_BLKSIZE(sbi) - sizeof(struct node_footer)); } -static inline struct f2fs_inode *F2FS_INODE(const struct folio *folio) +static inline struct f2fs_node *F2FS_NODE(const struct f2fs_cached_block *entry) { - return &((struct f2fs_node *)folio_address(folio))->i; + return (struct f2fs_node *)CACHED_NODE(entry); +} + +static inline struct f2fs_inode *F2FS_INODE(const struct f2fs_cached_block *entry) +{ + return &CACHED_NODE(entry)->i; +} + +static inline __le32 *F2FS_INODE_NIDS(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) +{ + return (__le32 *)(cache_address(entry) + + F2FS_BLKSIZE(sbi) - + sizeof(struct node_footer) - + SIZE_OF_I_NID); } static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi) @@ -2302,24 +2374,29 @@ static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi) return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info); } -static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi) +static inline bool f2fs_is_meta_cache(struct f2fs_cached_block *entry) { - return sbi->meta_inode->i_mapping; + return entry->cache && entry->cache == META_CACHE(entry->cache->sbi); } -static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi) +static inline bool f2fs_is_node_cache(struct f2fs_cached_block *entry) { - return sbi->node_inode->i_mapping; + return entry->cache && entry->cache == NODE_CACHE(entry->cache->sbi); } -static inline bool is_meta_folio(struct folio *folio) +static inline bool f2fs_is_compress_cache(struct f2fs_cached_block *entry) { - return folio->mapping == META_MAPPING(F2FS_F_SB(folio)); + return entry->cache && entry->cache == COMPRESS_CACHE(entry->cache->sbi); } -static inline bool is_node_folio(struct folio *folio) +static inline struct f2fs_bio *F2FS_BIO(struct bio *bio) { - return folio->mapping == NODE_MAPPING(F2FS_F_SB(folio)); + return container_of(bio, struct f2fs_bio, bio); +} + +static inline bool f2fs_is_cache_bio(struct bio *bio) +{ + return F2FS_BIO(bio)->entry != NULL; } static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type) @@ -2563,7 +2640,8 @@ static inline int F2FS_HAS_BLOCKS(struct inode *inode) { block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0; - return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block; + return (inode->i_blocks >> + F2FS_LOG_SECTORS_PER_BLOCK(F2FS_I_SB(inode))) > xattr_block; } static inline bool f2fs_has_xattr_block(unsigned int ofs) @@ -2713,17 +2791,14 @@ static inline void folio_clear_f2fs_##name(struct folio *folio) \ } F2FS_FOLIO_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); -F2FS_FOLIO_PRIVATE_GET_FUNC(inline, INLINE_INODE); F2FS_FOLIO_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); F2FS_FOLIO_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); F2FS_FOLIO_PRIVATE_SET_FUNC(reference, REF_RESOURCE); -F2FS_FOLIO_PRIVATE_SET_FUNC(inline, INLINE_INODE); F2FS_FOLIO_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); F2FS_FOLIO_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); F2FS_FOLIO_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); -F2FS_FOLIO_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); F2FS_FOLIO_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); F2FS_FOLIO_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); @@ -2751,7 +2826,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, struct inode *inode, block_t count) { - blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK; + blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK(sbi); spin_lock(&sbi->stat_lock); if (unlikely(sbi->total_valid_block_count < count)) { @@ -2778,9 +2853,9 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, f2fs_i_blocks_write(inode, count, false, true); } -static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) +static inline void inc_cache_count(struct f2fs_sb_info *sbi, int count_type) { - atomic_inc(&sbi->nr_pages[count_type]); + atomic_inc(&sbi->nr_caches[count_type]); if (count_type == F2FS_DIRTY_DENTS || count_type == F2FS_DIRTY_NODES || @@ -2793,15 +2868,15 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) static inline void inode_inc_dirty_pages(struct inode *inode) { atomic_inc(&F2FS_I(inode)->dirty_pages); - inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? + inc_cache_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); if (IS_NOQUOTA(inode)) - inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); + inc_cache_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); } -static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type) +static inline void dec_cache_count(struct f2fs_sb_info *sbi, int count_type) { - atomic_dec(&sbi->nr_pages[count_type]); + atomic_dec(&sbi->nr_caches[count_type]); } static inline void inode_dec_dirty_pages(struct inode *inode) @@ -2811,10 +2886,10 @@ static inline void inode_dec_dirty_pages(struct inode *inode) return; atomic_dec(&F2FS_I(inode)->dirty_pages); - dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? + dec_cache_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); if (IS_NOQUOTA(inode)) - dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); + dec_cache_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); } static inline void inc_atomic_write_cnt(struct inode *inode) @@ -2839,9 +2914,9 @@ static inline void release_atomic_write_cnt(struct inode *inode) fi->atomic_write_cnt = 0; } -static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type) +static inline s64 get_nr_caches(struct f2fs_sb_info *sbi, int count_type) { - return atomic_read(&sbi->nr_pages[count_type]); + return atomic_read(&sbi->nr_caches[count_type]); } static inline int get_dirty_pages(struct inode *inode) @@ -2851,7 +2926,7 @@ static inline int get_dirty_pages(struct inode *inode) static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type) { - return div_u64(get_pages(sbi, block_type) + BLKS_PER_SEC(sbi) - 1, + return div_u64(get_nr_caches(sbi, block_type) + BLKS_PER_SEC(sbi) - 1, BLKS_PER_SEC(sbi)); } @@ -2903,7 +2978,7 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) if (flag == NAT_BITMAP) return tmp_ptr; else - return (unsigned char *)ckpt + F2FS_BLKSIZE; + return (unsigned char *)ckpt + F2FS_BLKSIZE(sbi); } else { offset = (flag == NAT_BITMAP) ? le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0; @@ -3133,14 +3208,51 @@ static inline void f2fs_put_page(struct page *page, bool unlock) f2fs_folio_put(page_folio(page), unlock); } +#define F2FS_DENTRY_TAG_CACHE 1UL +#define F2FS_DENTRY_TAG_MASK 1UL + +static inline void *f2fs_cache_make_dentry_block(struct f2fs_cached_block *entry) +{ + if (IS_ERR_OR_NULL(entry)) + return entry; + return (void *)((unsigned long)entry | F2FS_DENTRY_TAG_CACHE); +} + +static inline bool f2fs_dentry_is_cache(void *dentry_block) +{ + return ((unsigned long)dentry_block & F2FS_DENTRY_TAG_MASK) == + F2FS_DENTRY_TAG_CACHE; +} + +static inline struct f2fs_cached_block *f2fs_dentry_cache(void *dentry_block) +{ + return (struct f2fs_cached_block *) + ((unsigned long)dentry_block & ~F2FS_DENTRY_TAG_MASK); +} + +static inline struct folio *f2fs_dentry_folio(void *dentry_block) +{ + return (struct folio *)dentry_block; +} + +static inline void f2fs_put_dentry_block(void *dentry_block, bool unlock) +{ + if (IS_ERR_OR_NULL(dentry_block)) + return; + if (f2fs_dentry_is_cache(dentry_block)) + f2fs_put_cache(f2fs_dentry_cache(dentry_block), unlock); + else + f2fs_folio_put(f2fs_dentry_folio(dentry_block), unlock); +} + static inline void f2fs_put_dnode(struct dnode_of_data *dn) { - if (dn->node_folio) - f2fs_folio_put(dn->node_folio, true); - if (dn->inode_folio && dn->node_folio != dn->inode_folio) - f2fs_folio_put(dn->inode_folio, false); - dn->node_folio = NULL; - dn->inode_folio = NULL; + if (dn->node_entry) + f2fs_put_cache(dn->node_entry, true); + if (dn->inode_entry && dn->node_entry != dn->inode_entry) + f2fs_put_cache(dn->inode_entry, false); + dn->node_entry = NULL; + dn->inode_entry = NULL; } static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name, @@ -3174,11 +3286,13 @@ static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep, static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type) { - if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) || - get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) || - get_pages(sbi, F2FS_WB_CP_DATA) || - get_pages(sbi, F2FS_DIO_READ) || - get_pages(sbi, F2FS_DIO_WRITE)) + if (get_nr_caches(sbi, F2FS_RD_DATA) || + get_nr_caches(sbi, F2FS_RD_NODE) || + get_nr_caches(sbi, F2FS_RD_META) || + get_nr_caches(sbi, F2FS_WB_DATA) || + get_nr_caches(sbi, F2FS_WB_CP_DATA) || + get_nr_caches(sbi, F2FS_DIO_READ) || + get_nr_caches(sbi, F2FS_DIO_WRITE)) return true; if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info && @@ -3193,7 +3307,8 @@ static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type) static inline bool is_inflight_read_io(struct f2fs_sb_info *sbi) { - return get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_DIO_READ); + return get_nr_caches(sbi, F2FS_RD_DATA) || + get_nr_caches(sbi, F2FS_DIO_READ); } static inline bool is_idle(struct f2fs_sb_info *sbi, int type) @@ -3229,13 +3344,11 @@ static inline void f2fs_radix_tree_insert(struct radix_tree_root *root, cond_resched(); } -#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino) - -static inline bool IS_INODE(const struct folio *folio) +static inline bool IS_INODE(struct f2fs_sb_info *sbi, const struct f2fs_cached_block *entry) { - struct f2fs_node *p = F2FS_NODE(folio); + struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry); - return RAW_IS_INODE(p); + return footer->nid == footer->ino; } static inline int offset_in_addr(struct f2fs_inode *i) @@ -3244,38 +3357,45 @@ static inline int offset_in_addr(struct f2fs_inode *i) (le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0; } -static inline __le32 *blkaddr_in_node(struct f2fs_node *node) +static inline __le32 *blkaddr_in_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr; + struct f2fs_node *node = F2FS_NODE(entry); + + return IS_INODE(sbi, entry) ? node->i.i_addr : node->dn.addr; } static inline int f2fs_has_extra_attr(struct inode *inode); + static inline unsigned int get_dnode_base(struct inode *inode, - struct folio *node_folio) + const struct f2fs_cached_block *entry) { - if (!IS_INODE(node_folio)) + if (!IS_INODE(inode ? F2FS_I_SB(inode) : entry->cache->sbi, entry)) return 0; return inode ? get_extra_isize(inode) : - offset_in_addr(&F2FS_NODE(node_folio)->i); + offset_in_addr(&CACHED_NODE(entry)->i); } static inline __le32 *get_dnode_addr(struct inode *inode, - struct folio *node_folio) + const struct f2fs_cached_block *entry) { - return blkaddr_in_node(F2FS_NODE(node_folio)) + - get_dnode_base(inode, node_folio); + struct f2fs_sb_info *sbi = inode ? F2FS_I_SB(inode) : entry->cache->sbi; + + return blkaddr_in_node(sbi, entry) + + get_dnode_base(inode, entry); } static inline block_t data_blkaddr(struct inode *inode, - struct folio *node_folio, unsigned int offset) + const struct f2fs_cached_block *entry, + unsigned int offset) { - return le32_to_cpu(*(get_dnode_addr(inode, node_folio) + offset)); + return le32_to_cpu(*(get_dnode_addr(inode, entry) + offset)); } static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn) { - return data_blkaddr(dn->inode, dn->node_folio, dn->ofs_in_node); + return data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node); } static inline int f2fs_test_bit(unsigned int nr, char *addr) @@ -3578,20 +3698,26 @@ static inline bool f2fs_need_compress_data(struct inode *inode) static inline unsigned int addrs_per_page(struct inode *inode, bool is_inode) { - unsigned int addrs = is_inode ? (CUR_ADDRS_PER_INODE(inode) - - get_inline_xattr_addrs(inode)) : DEF_ADDRS_PER_BLOCK; + unsigned int addrs = is_inode ? (cur_addrs_per_inode(inode) - + get_inline_xattr_addrs(inode)) : + DEF_ADDRS_PER_BLOCK(F2FS_I_SB(inode)); if (f2fs_compressed_file(inode)) return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size); return addrs; } -static inline -void *inline_xattr_addr(struct inode *inode, const struct folio *folio) +static inline unsigned int cur_addrs_per_inode(struct inode *inode) { - struct f2fs_inode *ri = F2FS_INODE(folio); + return DEF_ADDRS_PER_INODE(F2FS_I_SB(inode)) - get_extra_isize(inode); +} - return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE - +static inline void *inline_xattr_addr(struct inode *inode, + const struct f2fs_cached_block *entry) +{ + struct f2fs_inode *ri = F2FS_INODE(entry); + + return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE(F2FS_I_SB(inode)) - get_inline_xattr_addrs(inode)]); } @@ -3636,9 +3762,10 @@ static inline bool f2fs_is_cow_file(struct inode *inode) return is_inode_flag_set(inode, FI_COW_FILE); } -static inline void *inline_data_addr(struct inode *inode, struct folio *folio) +static inline void *inline_data_addr(struct inode *inode, + const struct f2fs_cached_block *entry) { - __le32 *addr = get_dnode_addr(inode, folio); + __le32 *addr = get_dnode_addr(inode, entry); return (void *)(addr + DEF_INLINE_RESERVED_SIZE); } @@ -3848,13 +3975,13 @@ int f2fs_pin_file_control(struct inode *inode, bool inc); * inode.c */ void f2fs_set_inode_flags(struct inode *inode); -bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio); -void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio); +bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry); +void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry); struct inode *f2fs_iget(struct super_block *sb, unsigned long ino); struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino); int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink); -void f2fs_update_inode(struct inode *inode, struct folio *node_folio); -void f2fs_update_inode_page(struct inode *inode); +void f2fs_update_inode(struct inode *inode, struct f2fs_cached_block *entry); +void f2fs_update_inode_cache(struct inode *inode); int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc); void f2fs_remove_donate_inode(struct inode *inode); void f2fs_evict_inode(struct inode *inode); @@ -3903,22 +4030,22 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, unsigned int start_pos, struct fscrypt_str *fstr); void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent, struct f2fs_dentry_ptr *d); -struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, - const struct f2fs_filename *fname, struct folio *dfolio); +struct f2fs_cached_block *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, + const struct f2fs_filename *fname, struct f2fs_cached_block *dentry); void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, unsigned int current_depth); int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots); void f2fs_drop_nlink(struct inode *dir, struct inode *inode); struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, - const struct f2fs_filename *fname, struct folio **res_folio); + const struct f2fs_filename *fname, void **dentry_block); struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, - const struct qstr *child, struct folio **res_folio); -struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f); + const struct qstr *child, void **dentry_block); +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, void **dentry_block); ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr, - struct folio **folio); + void **dentry_block); void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, - struct folio *folio, struct inode *inode); -bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio, + void *dentry_blk, struct inode *inode); +bool f2fs_has_enough_room(struct inode *dir, struct f2fs_cached_block *ientry, const struct f2fs_filename *fname); void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d, const struct fscrypt_str *name, f2fs_hash_t name_hash, @@ -3929,7 +4056,7 @@ int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname, struct inode *inode, nid_t ino, umode_t mode); int f2fs_do_add_link(struct inode *dir, const struct qstr *name, struct inode *inode, nid_t ino, umode_t mode); -void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, +void f2fs_delete_entry(struct f2fs_dir_entry *dentry, void *dentry_blk, struct inode *dir, struct inode *inode); int f2fs_do_tmpfile(struct inode *inode, struct inode *dir, struct f2fs_filename *fname); @@ -3951,7 +4078,7 @@ void f2fs_inode_synced(struct inode *inode); int f2fs_dquot_initialize(struct inode *inode); int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); int f2fs_do_quota_sync(struct super_block *sb, int type); -loff_t max_file_blocks(struct inode *inode); +loff_t max_file_blocks(struct f2fs_sb_info *sbi, struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag); void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error); @@ -3972,9 +4099,9 @@ enum node_type; int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid); bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type); -bool f2fs_in_warm_node_list(struct folio *folio); +bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry); void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi); -void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio); +void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry); void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi); bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid); bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid); @@ -3985,37 +4112,38 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs); int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode); int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from); int f2fs_truncate_xattr_node(struct inode *inode); -int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, +int f2fs_wait_on_node_caches_writeback(struct f2fs_sb_info *sbi, unsigned int seq_id); -int f2fs_remove_inode_page(struct inode *inode); -struct folio *f2fs_new_inode_folio(struct inode *inode); -struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); -void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); -struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, +int f2fs_write_node_caches(struct f2fs_sb_info *sbi, long *nr_written); +int f2fs_remove_inode_cache(struct inode *inode); +struct f2fs_cached_block *f2fs_new_inode_cache(struct inode *inode); +struct f2fs_cached_block *f2fs_new_node_cache(struct dnode_of_data *dn, unsigned int ofs); +void f2fs_ra_node_cache(struct f2fs_sb_info *sbi, nid_t nid); +struct f2fs_cached_block *f2fs_get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type); int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, - struct folio *folio, pgoff_t nid, + struct f2fs_cached_block *entry, pgoff_t nid, enum node_type ntype, bool in_irq); -struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); -struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); -int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, +struct f2fs_cached_block *f2fs_get_inode_cache(struct f2fs_sb_info *sbi, pgoff_t ino); +struct f2fs_cached_block *f2fs_get_xnode_cache(struct f2fs_sb_info *sbi, pgoff_t xnid); +int f2fs_write_node_cache(struct f2fs_cached_block *entry, int sync_mode, bool mark_dirty, enum iostat_type io_type); -int f2fs_move_node_folio(struct folio *node_folio, int gc_type); +int f2fs_move_node_cache(struct f2fs_cached_block *entry, int gc_type); void f2fs_flush_inline_data(struct f2fs_sb_info *sbi); -int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - struct writeback_control *wbc, bool atomic, - unsigned int *seq_id); -int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - struct writeback_control *wbc, - bool do_balance, enum iostat_type io_type); +int f2fs_fsync_node_caches(struct f2fs_sb_info *sbi, struct inode *inode, + bool atomic, unsigned int *seq_id); +int f2fs_writeback_node_caches(struct f2fs_sb_info *sbi, long nr_to_write, + bool sync, bool do_balance, enum iostat_type io_type, + long *nr_written); int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount); bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid); void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid); void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid); int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink); -int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio); -int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio); -int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio); +int f2fs_recover_inline_xattr(struct inode *inode, struct f2fs_cached_block *entry); +int f2fs_recover_xattr_data(struct inode *inode, struct f2fs_cached_block *entry); +int f2fs_recover_inode_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry); int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, unsigned int segno, struct f2fs_summary_block *sum); int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); @@ -4043,6 +4171,8 @@ void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr, bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); +void f2fs_drop_discard_cmd_range(struct f2fs_sb_info *sbi, + block_t start, block_t len); void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check); void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, @@ -4051,7 +4181,7 @@ void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi); int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable); void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi); -int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra); +int f2fs_nblocks_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra); bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno); int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi); int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi); @@ -4065,12 +4195,14 @@ int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi); int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range); bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, struct cp_control *cpc); -struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno); -void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src, +struct f2fs_cached_block *f2fs_get_sum_cache(struct f2fs_sb_info *sbi, + unsigned int segno); +void f2fs_update_meta_block(struct f2fs_sb_info *sbi, void *src, block_t blk_addr); -void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, - enum iostat_type io_type); -void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio); +void f2fs_do_write_meta_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, + enum iostat_type io_type); +void f2fs_do_write_node_cache(unsigned int nid, struct f2fs_io_info *fio); void f2fs_outplace_write_data(struct dnode_of_data *dn, struct f2fs_io_info *fio); int f2fs_inplace_write_data(struct f2fs_io_info *fio); @@ -4084,16 +4216,13 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, bool recover_newaddr); enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, enum log_type seg_type); -int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, +int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, block_t old_blkaddr, block_t *new_blkaddr, struct f2fs_summary *sum, int type, struct f2fs_io_info *fio); void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, block_t blkaddr, unsigned int blkcnt); -void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, - bool ordered, bool locked); -#define f2fs_wait_on_page_writeback(page, type, ordered, locked) \ - f2fs_folio_wait_writeback(page_folio(page), type, ordered, locked) +void f2fs_folio_wait_writeback(struct folio *folio, bool ordered, bool locked); void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr); void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, block_t len); @@ -4159,20 +4288,21 @@ void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, unsigned char reason); void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi); -struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index); -struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index); -struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index); -struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index); +struct f2fs_cached_block *f2fs_grab_meta_cache(struct f2fs_sb_info *sbi, pgoff_t index); +struct f2fs_cached_block *f2fs_get_meta_cache(struct f2fs_sb_info *sbi, pgoff_t index); +struct f2fs_cached_block *f2fs_get_meta_cache_retry(struct f2fs_sb_info *sbi, pgoff_t index); +struct f2fs_cached_block *f2fs_get_tmp_cache(struct f2fs_sb_info *sbi, pgoff_t index); bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr, int type); bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi, block_t blkaddr, int type); -int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, +int f2fs_ra_meta_caches(struct f2fs_sb_info *sbi, block_t start, int nrpages, int type, bool sync); -void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index, +void f2fs_ra_meta_caches_cond(struct f2fs_sb_info *sbi, pgoff_t index, unsigned int ra_blocks); -long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write, - enum iostat_type io_type); +void f2fs_write_meta_caches(struct f2fs_sb_info *sbi, long *nr_written); +long f2fs_sync_meta_caches(struct f2fs_sb_info *sbi, long nr_to_write, + bool sync, enum iostat_type io_type); void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type); void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type); void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all); @@ -4191,7 +4321,7 @@ void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio); void f2fs_remove_dirty_inode(struct inode *inode); int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, bool from_cp); -void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type); +void f2fs_sync_dirty_data(struct f2fs_sb_info *sbi, int type); u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi); int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi); @@ -4213,12 +4343,14 @@ void f2fs_destroy_bio_entry_cache(void); void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type); int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi); -void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type); void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, - struct inode *inode, struct folio *folio, - nid_t ino, enum page_type type); + struct inode *inode, struct folio *folio); void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, - struct folio *folio, enum page_type type); + struct folio *folio); +bool f2fs_submit_merged_write_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, + nid_t ino, enum page_type type); +void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type); void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, struct bio **bio, struct folio *folio); void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi); @@ -4226,6 +4358,8 @@ void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi); int f2fs_submit_page_bio(struct f2fs_io_info *fio); int f2fs_merge_page_bio(struct f2fs_io_info *fio); void f2fs_submit_page_write(struct f2fs_io_info *fio); +int f2fs_submit_cache_read(struct f2fs_io_info *fio); +void f2fs_submit_cache_write(struct f2fs_io_info *fio); struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, sector_t *sector); int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr); @@ -4242,7 +4376,7 @@ struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index, struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, bool for_write); struct folio *f2fs_get_new_data_folio(struct inode *inode, - struct folio *ifolio, pgoff_t index, bool new_i_size); + struct f2fs_cached_block *ientry, pgoff_t index, bool new_i_size); int f2fs_do_write_data_page(struct f2fs_io_info *fio); int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag); int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, @@ -4354,7 +4488,7 @@ struct f2fs_stat_info { unsigned int bimodal, avg_vblocks; int util_free, util_valid, util_invalid; int rsvd_segs, overp_segs; - int dirty_count, node_pages, meta_pages, compress_pages; + int dirty_count, node_caches, meta_caches, compress_pages; int compress_page_hit; int prefree_count, free_segs, free_secs; int cp_call_count[MAX_CALL_TYPE], cp_count; @@ -4377,6 +4511,8 @@ struct f2fs_stat_info { unsigned int block_count[2]; unsigned int inplace_count; unsigned long long base_mem, cache_mem, page_mem; + unsigned long long cache_entry_mem[NR_CACHE_TYPES]; + unsigned long long cache_data_mem[NR_CACHE_TYPES]; struct f2fs_dev_stats *dev_stats; }; @@ -4555,8 +4691,6 @@ extern const struct file_operations f2fs_dir_operations; extern const struct file_operations f2fs_file_operations; extern const struct inode_operations f2fs_file_inode_operations; extern const struct address_space_operations f2fs_dblock_aops; -extern const struct address_space_operations f2fs_node_aops; -extern const struct address_space_operations f2fs_meta_aops; extern const struct inode_operations f2fs_dir_inode_operations; extern const struct inode_operations f2fs_symlink_inode_operations; extern const struct inode_operations f2fs_encrypted_symlink_inode_operations; @@ -4567,26 +4701,27 @@ extern struct kmem_cache *f2fs_inode_entry_slab; * inline.c */ bool f2fs_may_inline_data(struct inode *inode); -bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio); +bool f2fs_sanity_check_inline_data(struct inode *inode, struct f2fs_cached_block *ientry); bool f2fs_may_inline_dentry(struct inode *inode); -void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio); -void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, +void f2fs_do_read_inline_data(struct folio *folio, struct f2fs_cached_block *ientry); +void f2fs_truncate_inline_inode(struct inode *inode, struct f2fs_cached_block *ientry, u64 from); int f2fs_read_inline_data(struct inode *inode, struct folio *folio); int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio); int f2fs_convert_inline_inode(struct inode *inode); int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry); int f2fs_write_inline_data(struct inode *inode, struct folio *folio); -int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio); +int f2fs_recover_inline_data(struct inode *inode, struct f2fs_cached_block *entry); struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir, - const struct f2fs_filename *fname, struct folio **res_folio, + const struct f2fs_filename *fname, void **dentry_block, bool use_hash); int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent, - struct folio *ifolio); + struct f2fs_cached_block *ientry); int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, struct inode *inode, nid_t ino, umode_t mode); void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, - struct folio *folio, struct inode *dir, struct inode *inode); + struct f2fs_cached_block *ientry, struct inode *dir, + struct inode *inode); bool f2fs_empty_inline_dir(struct inode *dir); int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, struct fscrypt_str *fstr); @@ -4609,7 +4744,7 @@ void f2fs_leave_shrinker(struct f2fs_sb_info *sbi); /* * extent_cache.c */ -bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio); +bool sanity_check_extent_cache(struct inode *inode, struct f2fs_cached_block *ientry); void f2fs_init_extent_tree(struct inode *inode); void f2fs_drop_extent_tree(struct inode *inode); void f2fs_destroy_extent_node(struct inode *inode); @@ -4619,7 +4754,7 @@ int __init f2fs_create_extent_cache(void); void f2fs_destroy_extent_cache(void); /* read extent cache ops */ -void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio); +void f2fs_init_read_extent_tree(struct inode *inode, struct f2fs_cached_block *ientry); bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, struct extent_info *ei); bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index, @@ -4741,13 +4876,11 @@ unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, int f2fs_init_compress_ctx(struct compress_ctx *cc); void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse); void f2fs_init_compress_info(struct f2fs_sb_info *sbi); -int f2fs_init_compress_inode(struct f2fs_sb_info *sbi); -void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi); +void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi); int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi); void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi); int __init f2fs_init_compress_cache(void); void f2fs_destroy_compress_cache(void); -struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi); void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, block_t blkaddr, unsigned int len); bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, @@ -4796,8 +4929,7 @@ static inline void f2fs_put_folio_dic(struct folio *folio, bool in_task) static inline unsigned int f2fs_cluster_blocks_are_contiguous( struct dnode_of_data *dn, unsigned int ofs_in_node) { return 0; } static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; } -static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; } -static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { } +static inline void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi) { } static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; } static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { } static inline int __init f2fs_init_compress_cache(void) { return 0; } @@ -5137,10 +5269,8 @@ static inline void f2fs_schedule_timeout_killable(long timeout, bool io) } static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi, - struct folio *folio, enum page_type type) + pgoff_t ofs, enum page_type type) { - pgoff_t ofs = folio->index; - if (unlikely(f2fs_cp_error(sbi))) return; @@ -5175,36 +5305,10 @@ static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi) return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb); } -static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi, - block_t blkaddr, unsigned int cnt) -{ - bool need_submit = false; - int i = 0; - - do { - struct folio *folio; - - folio = filemap_get_folio(META_MAPPING(sbi), blkaddr + i); - if (!IS_ERR(folio)) { - if (folio_test_writeback(folio)) - need_submit = true; - f2fs_folio_put(folio, false); - } - } while (++i < cnt && !need_submit); - - if (need_submit) - f2fs_submit_merged_write_cond(sbi, sbi->meta_inode, - NULL, 0, DATA); - - truncate_inode_pages_range(META_MAPPING(sbi), - F2FS_BLK_TO_BYTES((loff_t)blkaddr), - F2FS_BLK_END_BYTES((loff_t)(blkaddr + cnt - 1))); -} - static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi, block_t blkaddr, unsigned int len) { - f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); + f2fs_truncate_meta_caches(sbi, blkaddr, len); f2fs_invalidate_compress_pages_range(sbi, blkaddr, len); } diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 0c759cc99054..e2c0f5083313 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -110,7 +110,8 @@ static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); if (ret & VM_FAULT_LOCKED) f2fs_update_iostat(F2FS_I_SB(inode), inode, - APP_MAPPED_READ_IO, F2FS_BLKSIZE); + APP_MAPPED_READ_IO, + F2FS_BLKSIZE(F2FS_I_SB(inode))); trace_f2fs_filemap_fault(inode, vmf->pgoff, flags, ret); @@ -124,6 +125,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; bool need_alloc = !f2fs_is_pinned_file(inode); + loff_t isize; int err = 0; vm_fault_t ret; @@ -186,9 +188,26 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); + /* + * mmap stores bypass the COW inode and would break atomicity. Check + * under invalidate_lock: f2fs_ioc_start_atomic_write() holds it while + * flushing dirty folios and setting FI_ATOMIC_FILE, so either it sees + * our dirty folio and writes it back, or we see the flag. + */ + if (unlikely(f2fs_is_atomic_file(inode))) { + err = -EOPNOTSUPP; + goto out_sem; + } + folio_lock(folio); + /* + * A concurrent buffered write can extend isize, so we store isize in a + * local variable to let the zeroing offset match the EOF check below. + */ + isize = i_size_read(inode); + if (unlikely(folio->mapping != inode->i_mapping || - folio_pos(folio) > i_size_read(inode) || + folio_pos(folio) > isize || !folio_test_uptodate(folio))) { folio_unlock(folio); err = -EFAULT; @@ -212,28 +231,21 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) goto out_sem; } - f2fs_folio_wait_writeback(folio, DATA, false, true); + f2fs_folio_wait_writeback(folio, false, true); - /* wait for GCed page writeback via META_MAPPING */ + /* wait for GCed page writeback via generic cache */ f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); - /* - * check to see if the page is mapped already (no holes) - */ - if (folio_test_mappedtodisk(folio)) - goto out_sem; - /* page is wholly or partially inside EOF */ - if (((loff_t)(folio->index + 1) << PAGE_SHIFT) > - i_size_read(inode)) { + if (((loff_t)(folio->index + 1) << PAGE_SHIFT) > isize) { loff_t offset; - offset = i_size_read(inode) & ~PAGE_MASK; + offset = isize & ~PAGE_MASK; folio_zero_segment(folio, offset, folio_size(folio)); } folio_mark_dirty(folio); - f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE(sbi)); f2fs_update_time(sbi, REQ_TIME); out_sem: @@ -307,13 +319,13 @@ static inline enum cp_reason_type need_do_checkpoint(struct inode *inode) static bool need_inode_page_update(struct f2fs_sb_info *sbi, nid_t ino) { - struct folio *i = filemap_get_folio(NODE_MAPPING(sbi), ino); + struct f2fs_cached_block *entry = f2fs_find_node_cache(sbi, ino); bool ret = false; /* But we need to avoid that there are some inode updates */ - if ((!IS_ERR(i) && folio_test_dirty(i)) || + if ((!IS_ERR(entry) && f2fs_cache_test_dirty(entry)) || f2fs_need_inode_block_update(sbi, ino)) ret = true; - f2fs_folio_put(i, false); + f2fs_put_cache(entry, false); return ret; } @@ -339,10 +351,6 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end, nid_t ino = inode->i_ino; int ret = 0; enum cp_reason_type cp_reason = 0; - struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, - .nr_to_write = LONG_MAX, - }; unsigned int seq_id = 0; if (unlikely(f2fs_readonly(inode->i_sb))) @@ -421,7 +429,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end, } sync_nodes: atomic_inc(&sbi->wb_sync_req[NODE]); - ret = f2fs_fsync_node_pages(sbi, inode, &wbc, atomic, &seq_id); + ret = f2fs_fsync_node_caches(sbi, inode, atomic, &seq_id); atomic_dec(&sbi->wb_sync_req[NODE]); if (ret) goto out; @@ -447,7 +455,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end, * given fsync mark. */ if (!atomic) { - ret = f2fs_wait_on_node_pages_writeback(sbi, seq_id); + ret = f2fs_wait_on_node_caches_writeback(sbi, seq_id); if (ret) goto out; } @@ -484,7 +492,7 @@ static bool __found_offset(struct address_space *mapping, bool compressed_cluster = false; if (f2fs_compressed_file(inode)) { - block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, + block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_entry, ALIGN_DOWN(dn->ofs_in_node, F2FS_I(inode)->i_cluster_size)); compressed_cluster = first_blkaddr == COMPRESS_ADDR; @@ -513,7 +521,8 @@ static bool __found_offset(struct address_space *mapping, static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) { struct inode *inode = file->f_mapping->host; - loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + loff_t maxbytes = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), + max_file_blocks(F2FS_I_SB(inode), inode)); struct dnode_of_data dn; pgoff_t pgofs, end_offset; loff_t data_ofs = offset; @@ -537,9 +546,10 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) } } - pgofs = (pgoff_t)(offset >> PAGE_SHIFT); + pgofs = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode), offset); - for (; data_ofs < isize; data_ofs = (loff_t)pgofs << PAGE_SHIFT) { + for (; data_ofs < isize; + data_ofs = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), pgofs)) { set_new_dnode(&dn, inode, NULL, NULL, 0); err = f2fs_get_dnode_of_data(&dn, pgofs, LOOKUP_NODE); if (err && err != -ENOENT) { @@ -554,12 +564,12 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) } } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); /* find data/hole in dnode block */ for (; dn.ofs_in_node < end_offset; dn.ofs_in_node++, pgofs++, - data_ofs = (loff_t)pgofs << PAGE_SHIFT) { + data_ofs = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), pgofs)) { block_t blkaddr; blkaddr = f2fs_data_blkaddr(&dn); @@ -595,7 +605,8 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file->f_mapping->host; - loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + loff_t maxbytes = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), + max_file_blocks(F2FS_I_SB(inode), inode)); switch (whence) { case SEEK_SET: @@ -716,7 +727,7 @@ void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count) block_t blkstart; int blklen = 0; - addr = get_dnode_addr(dn->inode, dn->node_folio) + ofs; + addr = get_dnode_addr(dn->inode, dn->node_entry) + ofs; blkstart = le32_to_cpu(*addr); /* Assumption: truncation starts with cluster */ @@ -780,7 +791,7 @@ void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count) * once we invalidate valid blkaddr in range [ofs, ofs + count], * we will invalidate all blkaddr in the whole range. */ - fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), + fofs = f2fs_start_bidx_of_node(ofs_of_node(sbi, dn->node_entry), dn->inode) + ofs; f2fs_update_read_extent_cache_range(dn, fofs, 0, len); f2fs_update_age_extent_cache_range(dn, fofs, len); @@ -818,7 +829,7 @@ static int truncate_partial_data_page(struct inode *inode, u64 from, if (IS_ERR(folio)) return PTR_ERR(folio) == -ENOENT ? 0 : PTR_ERR(folio); truncate_out: - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); folio_zero_segment(folio, offset, folio_size(folio)); /* An encrypted inode should have a key and truncate the last page. */ @@ -836,7 +847,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) struct f2fs_lock_context lc; pgoff_t free_from; int count = 0, err = 0; - struct folio *ifolio; + struct f2fs_cached_block *ientry; bool truncate_page = false; trace_f2fs_truncate_blocks_enter(inode, from); @@ -846,17 +857,17 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) goto out_err; } - free_from = (pgoff_t)F2FS_BLK_ALIGN(from); + free_from = (pgoff_t)F2FS_BLK_ALIGN(sbi, from); - if (free_from >= max_file_blocks(inode)) + if (free_from >= max_file_blocks(sbi, inode)) goto free_partial; if (lock) f2fs_lock_op(sbi, &lc); - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) { - err = PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto out; } @@ -875,18 +886,18 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) f2fs_drop_extent_tree(inode); - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); goto out; } if (f2fs_has_inline_data(inode)) { - f2fs_truncate_inline_inode(inode, ifolio, from); - f2fs_folio_put(ifolio, true); + f2fs_truncate_inline_inode(inode, ientry, from); + f2fs_put_cache(ientry, true); truncate_page = true; goto out; } - set_new_dnode(&dn, inode, ifolio, NULL, 0); + set_new_dnode(&dn, inode, ientry, NULL, 0); err = f2fs_get_dnode_of_data(&dn, free_from, LOOKUP_NODE_RA); if (err) { if (err == -ENOENT) @@ -894,12 +905,12 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) goto out; } - count = ADDRS_PER_PAGE(dn.node_folio, inode); + count = ADDRS_PER_PAGE(dn.node_entry, inode); count -= dn.ofs_in_node; f2fs_bug_on(sbi, count < 0); - if (dn.ofs_in_node || IS_INODE(dn.node_folio)) { + if (dn.ofs_in_node || IS_INODE(sbi, dn.node_entry)) { f2fs_truncate_data_blocks_range(&dn, count); free_from += count; } @@ -959,9 +970,10 @@ int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock) int f2fs_truncate(struct inode *inode) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); int err; - if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || @@ -986,8 +998,8 @@ int f2fs_truncate(struct inode *inode) * leak in evict() path. */ truncate_inode_pages_range(inode->i_mapping, - F2FS_BLK_TO_BYTES(0), - F2FS_BLK_END_BYTES(0)); + F2FS_BLK_TO_BYTES(sbi, 0), + F2FS_BLK_END_BYTES(sbi, 0)); return err; } } @@ -1097,17 +1109,13 @@ static void __setattr_copy(const struct mnt_idmap *idmap, struct inode *inode, const struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; + struct iattr sattr = *attr; - i_uid_update(idmap, attr, inode); - i_gid_update(idmap, attr, inode); - if (ia_valid & ATTR_ATIME) - inode_set_atime_to_ts(inode, attr->ia_atime); - if (ia_valid & ATTR_MTIME) - inode_set_mtime_to_ts(inode, attr->ia_mtime); - if (ia_valid & ATTR_CTIME) - inode_set_ctime_to_ts(inode, attr->ia_ctime); + sattr.ia_valid &= ~ATTR_MODE; + + setattr_copy(idmap, inode, &sattr); if (ia_valid & ATTR_MODE) { - umode_t mode = attr->ia_mode; + umode_t mode = sattr.ia_mode; if (!in_group_or_capable(idmap, inode, i_gid_into_vfsgid(idmap, inode))) mode &= ~S_ISGID; @@ -1154,7 +1162,7 @@ int f2fs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, return -EOPNOTSUPP; if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) && !IS_ALIGNED(attr->ia_size, - F2FS_BLK_TO_BYTES(fi->i_cluster_size))) + F2FS_BLK_TO_BYTES(sbi, fi->i_cluster_size))) return -EINVAL; if (f2fs_is_pinned_file(inode)) { @@ -1170,7 +1178,7 @@ int f2fs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, * pinned file. */ else if (!IS_ALIGNED(attr->ia_size, - F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) + F2FS_BLK_TO_BYTES(sbi, CAP_BLKS_PER_SEC(sbi)))) return -EINVAL; } } @@ -1301,7 +1309,7 @@ static int fill_zero(struct inode *inode, pgoff_t index, if (IS_ERR(folio)) return PTR_ERR(folio); - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); folio_zero_range(folio, start, len); folio_mark_dirty(folio); f2fs_folio_put(folio, true); @@ -1327,7 +1335,7 @@ int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end) return err; } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); count = min(end_offset - dn.ofs_in_node, pg_end - pg_start); f2fs_bug_on(F2FS_I_SB(inode), count == 0 || count > end_offset); @@ -1427,7 +1435,7 @@ static int __read_out_blkaddrs(struct inode *inode, block_t *blkaddr, goto next; } - done = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, inode) - + done = min((pgoff_t)ADDRS_PER_PAGE(dn.node_entry, inode) - dn.ofs_in_node, len); for (i = 0; i < done; i++, blkaddr++, do_replace++, dn.ofs_in_node++) { *blkaddr = f2fs_data_blkaddr(&dn); @@ -1516,7 +1524,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, } ilen = min((pgoff_t) - ADDRS_PER_PAGE(dn.node_folio, dst_inode) - + ADDRS_PER_PAGE(dn.node_entry, dst_inode) - dn.ofs_in_node, len - i); do { dn.data_blkaddr = f2fs_data_blkaddr(&dn); @@ -1554,7 +1562,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, return PTR_ERR(fdst); } - f2fs_folio_wait_writeback(fdst, DATA, true, true); + f2fs_folio_wait_writeback(fdst, true, true); memcpy_folio(fdst, 0, fsrc, 0, PAGE_SIZE); folio_mark_dirty(fdst); @@ -1664,7 +1672,8 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) return -EINVAL; /* collapse range should be aligned to block size of f2fs. */ - if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + if (offset & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)) || + len & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode))) return -EINVAL; ret = f2fs_convert_inline_inode(inode); @@ -1823,7 +1832,7 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, goto out; } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); end = min(pg_end, end_offset - dn.ofs_in_node + index); ret = f2fs_do_zero_range(&dn, index, end); @@ -1879,7 +1888,8 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) return -EINVAL; /* insert range should be aligned to block size of f2fs. */ - if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + if (offset & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)) || + len & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode))) return -EINVAL; ret = f2fs_convert_inline_inode(inode); @@ -2107,9 +2117,11 @@ static long f2fs_fallocate(struct file *file, int mode, /* * Pinned file should not support partial truncation since the block - * can be used by applications. + * can be used by applications. Atomic files should not either, since + * these modify the original inode directly and break atomicity. */ - if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode)) && + if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode) || + f2fs_is_atomic_file(inode)) && (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE))) { ret = -EOPNOTSUPP; @@ -2397,6 +2409,8 @@ static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate) f2fs_down_write(&fi->i_gc_rwsem[WRITE]); f2fs_down_write(&fi->i_gc_rwsem[READ]); + /* serialize with f2fs_vm_page_mkwrite(), see the comment there */ + filemap_invalidate_lock(inode->i_mapping); /* * Should wait end_io to count F2FS_WB_CP_DATA correctly by @@ -2445,13 +2459,14 @@ static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate) fi->original_i_size = isize; if (truncate) { set_inode_flag(inode, FI_ATOMIC_REPLACE); - truncate_inode_pages_final(inode->i_mapping); + truncate_inode_pages(inode->i_mapping, 0); f2fs_i_size_write(inode, 0); isize = 0; } f2fs_i_size_write(fi->cow_inode, isize); out_unlock: + filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&fi->i_gc_rwsem[READ]); f2fs_up_write(&fi->i_gc_rwsem[WRITE]); if (ret) @@ -2557,7 +2572,7 @@ int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); break; case F2FS_GOING_DOWN_METAFLUSH: - f2fs_sync_meta_pages(sbi, LONG_MAX, FS_META_IO); + f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_META_IO); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); break; case F2FS_GOING_DOWN_NEED_FSCK: @@ -2641,7 +2656,8 @@ static int f2fs_keep_noreuse_range(struct inode *inode, loff_t offset, loff_t len) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - u64 max_bytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + u64 max_bytes = F2FS_BLK_TO_BYTES(sbi, + max_file_blocks(sbi, inode)); u64 start, end; int ret = 0; @@ -3029,8 +3045,8 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, /* writeback all dirty pages in the range */ err = filemap_write_and_wait_range(inode->i_mapping, - pg_start << PAGE_SHIFT, - (pg_end << PAGE_SHIFT) - 1); + (loff_t)pg_start << PAGE_SHIFT, + ((loff_t)pg_end << PAGE_SHIFT) - 1); if (err) goto out; @@ -3122,7 +3138,7 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, goto clear_out; } - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); folio_mark_dirty(folio); folio_set_f2fs_gcing(folio); @@ -3181,11 +3197,12 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) return -EFAULT; /* verify alignment of offset & size */ - if (range.start & (F2FS_BLKSIZE - 1) || range.len & (F2FS_BLKSIZE - 1)) + if (range.start & F2FS_BLKSIZE_MASK(sbi) || + range.len & F2FS_BLKSIZE_MASK(sbi)) return -EINVAL; - if (unlikely((range.start + range.len) >> PAGE_SHIFT > - max_file_blocks(inode))) + if (unlikely(F2FS_BYTES_TO_BLK(sbi, range.start + range.len) > + max_file_blocks(sbi, inode))) return -EINVAL; err = mnt_want_write_file(filp); @@ -3267,7 +3284,7 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (src == dst && pos_out > pos_in && pos_out < pos_in + len) goto out_unlock; if (pos_in + len == src->i_size) - len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in; + len = ALIGN(src->i_size, F2FS_BLKSIZE(sbi)) - pos_in; if (len == 0) { ret = 0; goto out_unlock; @@ -3285,9 +3302,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, } /* verify the end result is block aligned */ - if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) || - !IS_ALIGNED(pos_in + len, F2FS_BLKSIZE) || - !IS_ALIGNED(pos_out, F2FS_BLKSIZE)) + if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE(sbi)) || + !IS_ALIGNED(pos_in + len, F2FS_BLKSIZE(sbi)) || + !IS_ALIGNED(pos_out, F2FS_BLKSIZE(sbi))) goto out_unlock; ret = f2fs_convert_inline_inode(src); @@ -3319,9 +3336,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, } f2fs_lock_op(sbi, &lc); - ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(pos_in), - F2FS_BYTES_TO_BLK(pos_out), - F2FS_BYTES_TO_BLK(len), false); + ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(sbi, pos_in), + F2FS_BYTES_TO_BLK(sbi, pos_out), + F2FS_BYTES_TO_BLK(sbi, len), false); if (!ret) { if (dst_max_i_size) @@ -3840,10 +3857,11 @@ static int f2fs_ioc_reserve_dev_alias(struct file *filp) write_unlock(&et->lock); clear_inode_flag(inode, FI_NO_EXTENT); + f2fs_drop_discard_cmd_range(sbi, ei.blk, ei.len); f2fs_reserve_device_alias(sbi, ei.blk, ei.len); i_size_write(inode, (loff_t)ei.len << sbi->log_blocksize); - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); spin_lock(&FREE_I(sbi)->segmap_lock); FDEV(f2fs_target_device_index(sbi, ei.blk)).is_reserving = false; @@ -3937,7 +3955,7 @@ static int f2fs_ioc_release_dev_alias(struct file *filp) goto out_inode_unlock; } - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); f2fs_unlock_op(sbi, &lc); f2fs_up_write_trace(&sbi->gc_lock, &glc); @@ -3987,7 +4005,7 @@ int f2fs_precache_extents(struct inode *inode) map.m_next_extent = &m_next_extent; map.m_seg_type = NO_CHECK_TYPE; map.m_may_create = false; - end = F2FS_BLK_ALIGN(i_size_read(inode)); + end = F2FS_BLK_ALIGN(F2FS_I_SB(inode), i_size_read(inode)); while (map.m_lblk < end) { map.m_len = end - map.m_lblk; @@ -4156,7 +4174,7 @@ static int release_compress_blocks(struct dnode_of_data *dn, pgoff_t count) int i; for (i = 0; i < count; i++) { - blkaddr = data_blkaddr(dn->inode, dn->node_folio, + blkaddr = data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node + i); if (!__is_valid_data_blkaddr(blkaddr)) @@ -4275,7 +4293,7 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) break; } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); count = min(end_offset - dn.ofs_in_node, last_idx - page_idx); count = round_up(count, fi->i_cluster_size); @@ -4326,7 +4344,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, int i; for (i = 0; i < count; i++) { - blkaddr = data_blkaddr(dn->inode, dn->node_folio, + blkaddr = data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node + i); if (!__is_valid_data_blkaddr(blkaddr)) @@ -4343,7 +4361,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, int ret; for (i = 0; i < cluster_size; i++) { - blkaddr = data_blkaddr(dn->inode, dn->node_folio, + blkaddr = data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node + i); if (i == 0) { @@ -4454,7 +4472,7 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) break; } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); count = min(end_offset - dn.ofs_in_node, last_idx - page_idx); count = round_up(count, fi->i_cluster_size); @@ -4503,8 +4521,8 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) static int f2fs_secure_erase(struct block_device *bdev, struct inode *inode, pgoff_t off, block_t block, block_t len, u32 flags) { - sector_t sector = SECTOR_FROM_BLOCK(block); - sector_t nr_sects = SECTOR_FROM_BLOCK(len); + sector_t sector = SECTOR_FROM_BLOCK(F2FS_I_SB(inode), block); + sector_t nr_sects = SECTOR_FROM_BLOCK(F2FS_I_SB(inode), len); int ret = 0; if (flags & F2FS_TRIM_FILE_DISCARD) { @@ -4581,14 +4599,14 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) to_end = true; } - if (!IS_ALIGNED(range.start, F2FS_BLKSIZE) || - (!to_end && !IS_ALIGNED(end_addr, F2FS_BLKSIZE))) { + if (!IS_ALIGNED(range.start, F2FS_BLKSIZE(sbi)) || + (!to_end && !IS_ALIGNED(end_addr, F2FS_BLKSIZE(sbi)))) { ret = -EINVAL; goto err; } - index = F2FS_BYTES_TO_BLK(range.start); - pg_end = DIV_ROUND_UP(end_addr, F2FS_BLKSIZE); + index = F2FS_BYTES_TO_BLK(sbi, range.start); + pg_end = F2FS_BLK_ALIGN(sbi, end_addr); ret = f2fs_convert_inline_inode(inode); if (ret) @@ -4620,7 +4638,7 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) goto out; } - end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end_offset = ADDRS_PER_PAGE(dn.node_entry, inode); count = min(end_offset - dn.ofs_in_node, pg_end - index); for (i = 0; i < count; i++, index++, dn.ofs_in_node++) { struct block_device *cur_bdev; @@ -4816,7 +4834,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) /* It will never fail, when folio has pinned above */ f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(folio)); - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); folio_mark_dirty(folio); folio_set_f2fs_gcing(folio); @@ -5168,7 +5186,7 @@ static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error, { struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp)); - dec_page_count(sbi, F2FS_DIO_READ); + dec_cache_count(sbi, F2FS_DIO_READ); if (error) return error; f2fs_update_iostat(sbi, NULL, APP_DIRECT_READ_IO, size); @@ -5226,13 +5244,13 @@ static ssize_t f2fs_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) * the higher-level function iomap_dio_rw() in order to ensure that the * F2FS_DIO_READ counter will be decremented correctly in all cases. */ - inc_page_count(sbi, F2FS_DIO_READ); + inc_cache_count(sbi, F2FS_DIO_READ); dio = __iomap_dio_rw(iocb, to, &f2fs_iomap_ops, &f2fs_iomap_dio_read_ops, 0, NULL, 0); if (IS_ERR_OR_NULL(dio)) { ret = PTR_ERR_OR_ZERO(dio); if (ret != -EIOCBQUEUED) - dec_page_count(sbi, F2FS_DIO_READ); + dec_cache_count(sbi, F2FS_DIO_READ); } else { ret = iomap_dio_complete(dio); } @@ -5285,7 +5303,7 @@ static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) /* In LFS mode, if there is inflight dio, wait for its completion */ if (f2fs_lfs_mode(F2FS_I_SB(inode)) && - get_pages(F2FS_I_SB(inode), F2FS_DIO_WRITE) && + get_nr_caches(F2FS_I_SB(inode), F2FS_DIO_WRITE) && (!f2fs_is_pinned_file(inode) || !dio)) inode_dio_wait(inode); @@ -5378,7 +5396,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter, * buffered IO, if DIO meets any holes. */ if (dio && i_size_read(inode) && - (F2FS_BYTES_TO_BLK(pos) < F2FS_BLK_ALIGN(i_size_read(inode)))) + (F2FS_BYTES_TO_BLK(sbi, pos) < + F2FS_BLK_ALIGN(sbi, i_size_read(inode)))) return 0; /* No-wait I/O can't allocate blocks. */ @@ -5399,8 +5418,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter, } /* Do not preallocate blocks that will be written partially in 4KB. */ - map.m_lblk = F2FS_BLK_ALIGN(pos); - map.m_len = F2FS_BYTES_TO_BLK(pos + count); + map.m_lblk = F2FS_BLK_ALIGN(sbi, pos); + map.m_len = F2FS_BYTES_TO_BLK(sbi, pos + count); if (map.m_len > map.m_lblk) map.m_len -= map.m_lblk; else @@ -5450,7 +5469,7 @@ static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, { struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp)); - dec_page_count(sbi, F2FS_DIO_WRITE); + dec_cache_count(sbi, F2FS_DIO_WRITE); if (error) return error; f2fs_update_time(sbi, REQ_TIME); @@ -5561,7 +5580,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, * the higher-level function iomap_dio_rw() in order to ensure that the * F2FS_DIO_WRITE counter will be decremented correctly in all cases. */ - inc_page_count(sbi, F2FS_DIO_WRITE); + inc_cache_count(sbi, F2FS_DIO_WRITE); dio_flags = 0; if (pos + count > inode->i_size) dio_flags |= IOMAP_DIO_FORCE_WAIT; @@ -5572,7 +5591,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, if (ret == -ENOTBLK) ret = 0; if (ret != -EIOCBQUEUED) - dec_page_count(sbi, F2FS_DIO_WRITE); + dec_cache_count(sbi, F2FS_DIO_WRITE); } else { ret = iomap_dio_complete(dio); } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 0a00180c21dc..5b948c92f981 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1054,7 +1054,7 @@ static int gc_node_segment(struct f2fs_sb_info *sbi, for (off = 0; off < usable_blks_in_seg; off++, entry++) { nid_t nid = le32_to_cpu(entry->nid); - struct folio *node_folio; + struct f2fs_cached_block *node_entry; struct node_info ni; int err; @@ -1066,38 +1066,38 @@ static int gc_node_segment(struct f2fs_sb_info *sbi, continue; if (phase == 0) { - f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1, + f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), 1, META_NAT, true); continue; } if (phase == 1) { - f2fs_ra_node_page(sbi, nid); + f2fs_ra_node_cache(sbi, nid); continue; } /* phase == 2 */ - node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); - if (IS_ERR(node_folio)) + node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_entry)) continue; /* block may become invalid during f2fs_get_node_folio */ if (check_valid_map(sbi, segno, off) == 0) { - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); continue; } if (f2fs_get_node_info(sbi, nid, &ni, false)) { - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); continue; } if (ni.blk_addr != start_addr + off) { - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); continue; } - err = f2fs_move_node_folio(node_folio, gc_type); + err = f2fs_move_node_cache(node_entry, gc_type); if (!err && gc_type == FG_GC) submitted++; stat_inc_node_blk_count(sbi, 1, gc_type); @@ -1123,7 +1123,8 @@ static int gc_node_segment(struct f2fs_sb_info *sbi, */ block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode) { - unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int indirect_blks = 2 * NIDS_PER_BLOCK(sbi) + 4; unsigned int bidx; if (node_ofs == 0) @@ -1132,11 +1133,12 @@ block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode) if (node_ofs <= 2) { bidx = node_ofs - 1; } else if (node_ofs <= indirect_blks) { - int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1); + int dec = (node_ofs - 4) / (NIDS_PER_BLOCK(sbi) + 1); bidx = node_ofs - 2 - dec; } else { - int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1); + int dec = (node_ofs - indirect_blks - 3) / + (NIDS_PER_BLOCK(sbi) + 1); bidx = node_ofs - 5 - dec; } @@ -1146,7 +1148,7 @@ block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode) static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, struct node_info *dni, block_t blkaddr, unsigned int *nofs) { - struct folio *node_folio; + struct f2fs_cached_block *node_entry; nid_t nid; unsigned int ofs_in_node, max_addrs, base; block_t source_blkaddr; @@ -1154,12 +1156,12 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, nid = le32_to_cpu(sum->nid); ofs_in_node = le16_to_cpu(sum->ofs_in_node); - node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); - if (IS_ERR(node_folio)) + node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_entry)) return false; if (f2fs_get_node_info(sbi, nid, dni, false)) { - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); return false; } @@ -1170,28 +1172,28 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } if (f2fs_check_nid_range(sbi, dni->ino)) { - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); return false; } - if (IS_INODE(node_folio)) { - base = offset_in_addr(F2FS_INODE(node_folio)); - max_addrs = DEF_ADDRS_PER_INODE; + if (IS_INODE(sbi, node_entry)) { + base = offset_in_addr(F2FS_INODE(node_entry)); + max_addrs = DEF_ADDRS_PER_INODE(sbi); } else { base = 0; - max_addrs = DEF_ADDRS_PER_BLOCK; + max_addrs = DEF_ADDRS_PER_BLOCK(sbi); } if (base + ofs_in_node >= max_addrs) { f2fs_err(sbi, "Inconsistent blkaddr offset: base:%u, ofs_in_node:%u, max:%u, ino:%u, nid:%u", base, ofs_in_node, max_addrs, dni->ino, dni->nid); - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); return false; } - *nofs = ofs_of_node(node_folio); - source_blkaddr = data_blkaddr(NULL, node_folio, ofs_in_node); - f2fs_folio_put(node_folio, true); + *nofs = ofs_of_node(sbi, node_entry); + source_blkaddr = data_blkaddr(NULL, node_entry, ofs_in_node); + f2fs_put_cache(node_entry, true); if (source_blkaddr != blkaddr) { #ifdef CONFIG_F2FS_CHECK_FS @@ -1217,7 +1219,8 @@ static int ra_data_block(struct inode *inode, pgoff_t index) struct address_space *mapping = inode->i_mapping; struct inode *atomic_inode = NULL; struct dnode_of_data dn; - struct folio *folio, *efolio; + struct folio *folio; + struct f2fs_cached_block *entry; struct f2fs_io_info fio = { .sbi = sbi, .ino = inode->i_ino, @@ -1227,6 +1230,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) .op_flags = 0, .encrypted_page = NULL, .in_list = 0, + .is_cache = 1, }; int err = 0; @@ -1281,36 +1285,35 @@ static int ra_data_block(struct inode *inode, pgoff_t index) * don't cache encrypted data into meta inode until previous dirty * data were writebacked to avoid racing between GC and flush. */ - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); - efolio = f2fs_filemap_get_folio(META_MAPPING(sbi), dn.data_blkaddr, - FGP_LOCK | FGP_CREAT, GFP_NOFS); - if (IS_ERR(efolio)) { - err = PTR_ERR(efolio); + entry = f2fs_grab_cache(META_CACHE(sbi), dn.data_blkaddr, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); goto put_folio; } - fio.encrypted_page = &efolio->page; + if (f2fs_cache_test_uptodate(entry)) + goto put_cache; - if (folio_test_uptodate(efolio)) - goto put_encrypted_page; - - err = f2fs_submit_page_bio(&fio); + fio.cache_entry = entry; + err = f2fs_submit_cache_read(&fio); if (err) - goto put_encrypted_page; - f2fs_put_page(fio.encrypted_page, false); + goto put_cache; + f2fs_put_cache(entry, false); f2fs_folio_put(folio, true); - f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); - f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE(sbi)); + f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE(sbi)); if (atomic_inode) iput(atomic_inode); return 0; -put_encrypted_page: - f2fs_put_page(fio.encrypted_page, true); +put_cache: + f2fs_put_cache(entry, true); put_folio: f2fs_folio_put(folio, true); out_iput: @@ -1320,7 +1323,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) } /* - * Move data block via META_MAPPING while keeping locked data page. + * Move data block via meta cache while keeping locked data page. * This can be used to move blocks, aka LBAs, directly on disk. */ static int move_data_block(struct inode *inode, block_t bidx, @@ -1337,11 +1340,13 @@ static int move_data_block(struct inode *inode, block_t bidx, .op_flags = 0, .encrypted_page = NULL, .in_list = 0, + .is_cache = 1, }; struct dnode_of_data dn; struct f2fs_summary sum; struct node_info ni; - struct folio *folio, *mfolio, *efolio; + struct folio *folio; + struct f2fs_cached_block *sentry, *tentry; block_t newaddr; int err = 0; bool lfs_mode = f2fs_lfs_mode(fio.sbi); @@ -1391,7 +1396,7 @@ static int move_data_block(struct inode *inode, block_t bidx, * don't cache encrypted data into meta inode until previous dirty * data were writebacked to avoid racing between GC and flush. */ - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); @@ -1406,33 +1411,33 @@ static int move_data_block(struct inode *inode, block_t bidx, if (lfs_mode) f2fs_down_write(&fio.sbi->io_order_lock); - mfolio = f2fs_grab_cache_folio(META_MAPPING(fio.sbi), - fio.old_blkaddr, false); - if (IS_ERR(mfolio)) { - err = PTR_ERR(mfolio); + sentry = f2fs_grab_cache(META_CACHE(fio.sbi), fio.old_blkaddr, + F2FS_CACHE_LOCK_CREATE); + if (IS_ERR(sentry)) { + err = PTR_ERR(sentry); goto up_out; } - fio.encrypted_page = folio_file_page(mfolio, fio.old_blkaddr); + fio.cache_entry = sentry; - /* read source block in mfolio */ - if (!folio_test_uptodate(mfolio)) { - err = f2fs_submit_page_bio(&fio); + /* read source block in cache */ + if (!f2fs_cache_test_uptodate(sentry)) { + err = f2fs_submit_cache_read(&fio); if (err) { - f2fs_folio_put(mfolio, true); + f2fs_put_cache(sentry, true); goto up_out; } f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO, - F2FS_BLKSIZE); + F2FS_BLKSIZE(fio.sbi)); f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO, - F2FS_BLKSIZE); + F2FS_BLKSIZE(fio.sbi)); - folio_lock(mfolio); - if (unlikely(!is_meta_folio(mfolio) || - !folio_test_uptodate(mfolio))) { + f2fs_lock_cache(sentry); + if (unlikely(!f2fs_is_meta_cache(sentry) || + !f2fs_cache_test_uptodate(sentry))) { err = -EIO; - f2fs_folio_put(mfolio, true); + f2fs_put_cache(sentry, true); goto up_out; } } @@ -1440,49 +1445,58 @@ static int move_data_block(struct inode *inode, block_t bidx, set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version); /* allocate block address */ - err = f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr, + err = f2fs_allocate_data_block(fio.sbi, fio.old_blkaddr, &newaddr, &sum, type, NULL); if (err) { - f2fs_folio_put(mfolio, true); + f2fs_put_cache(sentry, true); /* filesystem should shutdown, no need to recovery block */ goto up_out; } - efolio = f2fs_filemap_get_folio(META_MAPPING(fio.sbi), newaddr, - FGP_LOCK | FGP_CREAT, GFP_NOFS); - if (IS_ERR(efolio)) { - err = PTR_ERR(efolio); - f2fs_folio_put(mfolio, true); + /* + * If newaddr is in a sequential zone, it must be written once it is + * allocated, otherwise it leaves a hole in the zone, and the following + * write to the zone will fail as an unaligned write, so don't fail + * here. + */ + tentry = f2fs_grab_cache(META_CACHE(fio.sbi), newaddr, + F2FS_CACHE_LOCK_CREATE | + (f2fs_is_sequential_zone_area(fio.sbi, newaddr) ? + F2FS_CACHE_NOFAIL : 0)); + if (IS_ERR(tentry)) { + err = PTR_ERR(tentry); + f2fs_put_cache(sentry, true); goto recover_block; } - fio.encrypted_page = &efolio->page; + fio.cache_entry = tentry; /* write target block */ - f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true, true); - memcpy(page_address(fio.encrypted_page), - folio_address(mfolio), PAGE_SIZE); - f2fs_folio_put(mfolio, true); + f2fs_cache_wait_writeback_cond(tentry, DATA); + memcpy(cache_address(tentry), cache_address(sentry), + fio.sbi->blocksize); + f2fs_put_cache(sentry, true); f2fs_invalidate_internal_cache(fio.sbi, fio.old_blkaddr, 1); - set_page_dirty(fio.encrypted_page); - if (clear_page_dirty_for_io(fio.encrypted_page)) - dec_page_count(fio.sbi, F2FS_DIRTY_META); + f2fs_mark_cache_dirty(tentry); + if (f2fs_cache_test_and_clear_dirty(tentry)) + dec_cache_count(fio.sbi, F2FS_DIRTY_META); - set_page_writeback(fio.encrypted_page); + f2fs_start_cache_writeback(tentry); fio.op = REQ_OP_WRITE; fio.op_flags = REQ_SYNC; fio.new_blkaddr = newaddr; - f2fs_submit_page_write(&fio); + f2fs_submit_cache_write(&fio); - f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE); + f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, + F2FS_BLKSIZE(fio.sbi)); f2fs_update_data_blkaddr(&dn, newaddr); set_inode_flag(inode, FI_APPEND_WRITE); - f2fs_put_page(fio.encrypted_page, true); + f2fs_put_cache(tentry, true); recover_block: if (err) f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr, @@ -1543,7 +1557,7 @@ static int move_data_page(struct inode *inode, block_t bidx, int gc_type, bool is_dirty = folio_test_dirty(folio); retry: - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); folio_mark_dirty(folio); if (folio_clear_dirty_for_io(folio)) { @@ -1614,13 +1628,13 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, continue; if (phase == 0) { - f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1, + f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), 1, META_NAT, true); continue; } if (phase == 1) { - f2fs_ra_node_page(sbi, nid); + f2fs_ra_node_cache(sbi, nid); continue; } @@ -1629,7 +1643,7 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, continue; if (phase == 2) { - f2fs_ra_node_page(sbi, dni.ino); + f2fs_ra_node_cache(sbi, dni.ino); continue; } @@ -1818,28 +1832,31 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); /* readahead multi ssa blocks those have contiguous address */ if (__is_large_section(sbi)) - f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), + f2fs_ra_meta_caches(sbi, GET_SUM_BLOCK(sbi, segno), sum_blk_cnt, META_SSA, true); /* reference all summary page */ while (segno < end_segno) { - struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); + struct f2fs_cached_block *sum_entry = + f2fs_get_sum_cache(sbi, segno); segno += sbi->sums_per_block; - if (IS_ERR(sum_folio)) { - int err = PTR_ERR(sum_folio); + if (IS_ERR(sum_entry)) { + int err = PTR_ERR(sum_entry); end_segno = segno - sbi->sums_per_block; segno = rounddown(start_segno, sbi->sums_per_block); while (segno < end_segno) { - sum_folio = filemap_get_folio(META_MAPPING(sbi), + sum_entry = f2fs_find_meta_cache(sbi, GET_SUM_BLOCK(sbi, segno)); - folio_put_refs(sum_folio, 2); + f2fs_put_cache(sum_entry, false); + f2fs_put_cache(sum_entry, false); segno += sbi->sums_per_block; } return err; } - folio_unlock(sum_folio); + f2fs_unlock_cache(sum_entry); + } blk_start_plug(&plug); @@ -1847,16 +1864,24 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, segno = start_segno; while (segno < end_segno) { unsigned int cur_segno; + unsigned int block_end_segno; /* find segment summary of victim */ - struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), + struct f2fs_cached_block *sum_entry = + f2fs_find_meta_cache(sbi, GET_SUM_BLOCK(sbi, segno)); - unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + f2fs_bug_on(sbi, IS_ERR(sum_entry)); + + block_end_segno = rounddown(segno, sbi->sums_per_block) + sbi->sums_per_block; if (block_end_segno > end_segno) block_end_segno = end_segno; + if (unlikely(freezing(current))) + goto next_block; + if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) { f2fs_err(sbi, "%s: segment %u is used by log", __func__, segno); @@ -1864,8 +1889,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, goto next_block; } - if (!folio_test_uptodate(sum_folio) || - unlikely(f2fs_cp_error(sbi))) + if (!f2fs_cache_test_uptodate(sum_entry) || + unlikely(f2fs_cp_error(sbi))) goto next_block; for (cur_segno = segno; cur_segno < block_end_segno; @@ -1884,7 +1909,7 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, data_type = (type == SUM_TYPE_DATA) ? DATA : NODE; } - sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + sum = SUM_BLK_ENTRY_ADDR(sbi, sum_entry, cur_segno); if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { f2fs_err(sbi, "Inconsistent segment (%u) type " "[%d, %d] in SIT and SSA", @@ -1924,18 +1949,13 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sbi->next_victim_seg[gc_type] = (cur_segno + 1 < sec_end_segno) ? cur_segno + 1 : NULL_SEGNO; - - if (unlikely(freezing(current))) { - folio_put_refs(sum_folio, 2); - goto stop; - } } next_block: - folio_put_refs(sum_folio, 2); + f2fs_put_cache(sum_entry, false); + f2fs_put_cache(sum_entry, false); segno = block_end_segno; } -stop: if (submitted) f2fs_submit_merged_write(sbi, data_type); @@ -1963,9 +1983,9 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) trace_f2fs_gc_begin(sbi->sb, gc_type, gc_control->no_bg_gc, gc_control->nr_free_secs, - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DENTS), - get_pages(sbi, F2FS_DIRTY_IMETA), + get_nr_caches(sbi, F2FS_DIRTY_NODES), + get_nr_caches(sbi, F2FS_DIRTY_DENTS), + get_nr_caches(sbi, F2FS_DIRTY_IMETA), free_sections(sbi), free_segments(sbi), reserved_segments(sbi), @@ -2089,9 +2109,9 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) f2fs_unpin_all_sections(sbi, true); trace_f2fs_gc_end(sbi->sb, ret, total_freed, total_sec_freed, - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DENTS), - get_pages(sbi, F2FS_DIRTY_IMETA), + get_nr_caches(sbi, F2FS_DIRTY_NODES), + get_nr_caches(sbi, F2FS_DIRTY_DENTS), + get_nr_caches(sbi, F2FS_DIRTY_IMETA), free_sections(sbi), free_segments(sbi), reserved_segments(sbi), @@ -2326,6 +2346,7 @@ static void update_fs_metadata(struct f2fs_sb_info *sbi, int secs) SM_I(sbi)->segment_count = (int)SM_I(sbi)->segment_count + segs; MAIN_SEGS(sbi) = (int)MAIN_SEGS(sbi) + segs; MAIN_SECS(sbi) += secs; + f2fs_adjust_pinned_area_boundary(sbi); if (sbi->allocate_section_hint > MAIN_SECS(sbi)) sbi->allocate_section_hint = MAIN_SECS(sbi); FREE_I(sbi)->free_sections = (int)FREE_I(sbi)->free_sections + secs; @@ -2357,7 +2378,7 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) struct f2fs_lock_context lc; struct f2fs_lock_context glc; struct f2fs_lock_context clc; - unsigned int secs; + unsigned int secs, old_tail; int err = 0; __u32 rem; @@ -2466,6 +2487,8 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) goto recover_out; } + old_tail = F2FS_OPTION(sbi).resizable_tail_secno; + F2FS_OPTION(sbi).resizable_tail_secno -= min(old_tail, secs); update_fs_metadata(sbi, -secs); clear_sbi_flag(sbi, SBI_IS_RESIZEFS); set_sbi_flag(sbi, SBI_IS_DIRTY); @@ -2473,6 +2496,7 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) stat_inc_cp_call_count(sbi, TOTAL_CALL); err = f2fs_write_checkpoint(sbi, &cpc); if (err) { + F2FS_OPTION(sbi).resizable_tail_secno = old_tail; update_fs_metadata(sbi, secs); update_sb_metadata(sbi, secs); f2fs_commit_super(sbi, false); diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index aec06fb4fd76..273b6428c9f9 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -34,27 +34,26 @@ bool f2fs_may_inline_data(struct inode *inode) return !f2fs_post_read_required(inode); } -static bool inode_has_blocks(struct inode *inode, struct folio *ifolio) +static bool inode_has_blocks(struct inode *inode, struct f2fs_cached_block *ientry) { - struct f2fs_inode *ri = F2FS_INODE(ifolio); int i; if (F2FS_HAS_BLOCKS(inode)) return true; for (i = 0; i < DEF_NIDS_PER_INODE; i++) { - if (ri->i_nid[i]) + if (F2FS_INODE_NIDS(F2FS_I_SB(inode), ientry)[i]) return true; } return false; } -bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio) +bool f2fs_sanity_check_inline_data(struct inode *inode, struct f2fs_cached_block *ientry) { if (!f2fs_has_inline_data(inode)) return false; - if (inode_has_blocks(inode, ifolio)) + if (inode_has_blocks(inode, ientry)) return false; if (!support_inline_data(inode)) @@ -80,7 +79,7 @@ bool f2fs_may_inline_dentry(struct inode *inode) return true; } -void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio) +void f2fs_do_read_inline_data(struct folio *folio, struct f2fs_cached_block *ientry) { struct inode *inode = folio->mapping->host; @@ -92,13 +91,13 @@ void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio) folio_zero_segment(folio, MAX_INLINE_DATA(inode), folio_size(folio)); /* Copy the whole inline data block */ - memcpy_to_folio(folio, 0, inline_data_addr(inode, ifolio), + memcpy_to_folio(folio, 0, inline_data_addr(inode, ientry), MAX_INLINE_DATA(inode)); if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); } -void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, +void f2fs_truncate_inline_inode(struct inode *inode, struct f2fs_cached_block *ientry, u64 from) { void *addr; @@ -106,11 +105,11 @@ void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, if (from >= MAX_INLINE_DATA(inode)) return; - addr = inline_data_addr(inode, ifolio); + addr = inline_data_addr(inode, ientry); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); memset(addr + from, 0, MAX_INLINE_DATA(inode) - from); - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); if (from == 0) clear_inode_flag(inode, FI_DATA_EXIST); @@ -118,27 +117,27 @@ void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, int f2fs_read_inline_data(struct inode *inode, struct folio *folio) { - struct folio *ifolio; + struct f2fs_cached_block *ientry; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) { + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) { folio_unlock(folio); - return PTR_ERR(ifolio); + return PTR_ERR(ientry); } if (!f2fs_has_inline_data(inode)) { - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return -EAGAIN; } if (folio->index) folio_zero_segment(folio, 0, folio_size(folio)); else - f2fs_do_read_inline_data(folio, ifolio); + f2fs_do_read_inline_data(folio, ientry); if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); folio_unlock(folio); return 0; } @@ -186,7 +185,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) f2fs_bug_on(F2FS_F_SB(folio), folio_test_writeback(folio)); - f2fs_do_read_inline_data(folio, dn->inode_folio); + f2fs_do_read_inline_data(folio, dn->inode_entry); folio_mark_dirty(folio); /* clear dirty state */ @@ -197,7 +196,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) fio.old_blkaddr = dn->data_blkaddr; set_inode_flag(dn->inode, FI_HOT_DATA); f2fs_outplace_write_data(dn, &fio); - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); if (dirty) { inode_dec_dirty_pages(dn->inode); f2fs_remove_dirty_inode(dn->inode); @@ -207,8 +206,8 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) set_inode_flag(dn->inode, FI_APPEND_WRITE); /* clear inline data and flag after data writeback */ - f2fs_truncate_inline_inode(dn->inode, dn->inode_folio, 0); - folio_clear_f2fs_inline(dn->inode_folio); + f2fs_truncate_inline_inode(dn->inode, dn->inode_entry, 0); + f2fs_cache_clear_inline(dn->inode_entry); clear_out: stat_dec_inline_inode(dn->inode); clear_inode_flag(dn->inode, FI_INLINE_DATA); @@ -221,7 +220,8 @@ int f2fs_convert_inline_inode(struct inode *inode) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; struct f2fs_lock_context lc; - struct folio *ifolio, *folio; + struct f2fs_cached_block *ientry; + struct folio *folio; int err = 0; if (f2fs_hw_is_readonly(sbi) || f2fs_readonly(sbi->sb)) @@ -240,13 +240,13 @@ int f2fs_convert_inline_inode(struct inode *inode) f2fs_lock_op(sbi, &lc); - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) { - err = PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto out; } - set_new_dnode(&dn, inode, ifolio, ifolio, 0); + set_new_dnode(&dn, inode, ientry, ientry, 0); if (f2fs_has_inline_data(inode)) err = f2fs_convert_inline_folio(&dn, folio); @@ -266,35 +266,35 @@ int f2fs_convert_inline_inode(struct inode *inode) int f2fs_write_inline_data(struct inode *inode, struct folio *folio) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct folio *ifolio; + struct f2fs_cached_block *ientry; - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); if (!f2fs_has_inline_data(inode)) { - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return -EAGAIN; } f2fs_bug_on(F2FS_I_SB(inode), folio->index); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); - memcpy_from_folio(inline_data_addr(inode, ifolio), + f2fs_cache_wait_writeback(ientry); + memcpy_from_folio(inline_data_addr(inode, ientry), folio, 0, MAX_INLINE_DATA(inode)); - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); f2fs_clear_page_cache_dirty_tag(folio); set_inode_flag(inode, FI_APPEND_WRITE); set_inode_flag(inode, FI_DATA_EXIST); - folio_clear_f2fs_inline(ifolio); - f2fs_folio_put(ifolio, true); + f2fs_cache_clear_inline(ientry); + f2fs_put_cache(ientry, true); return 0; } -int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio) +int f2fs_recover_inline_data(struct inode *inode, struct f2fs_cached_block *entry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode *ri = NULL; @@ -308,39 +308,41 @@ int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio) * x o -> remove data blocks, and then recover inline_data * x x -> recover data blocks */ - if (IS_INODE(nfolio)) - ri = F2FS_INODE(nfolio); + if (IS_INODE(F2FS_I_SB(inode), entry)) + ri = &CACHED_NODE(entry)->i; if (f2fs_has_inline_data(inode) && ri && (ri->i_inline & F2FS_INLINE_DATA)) { - struct folio *ifolio; + struct f2fs_cached_block *ientry; + process_inline: - ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); - src_addr = inline_data_addr(inode, nfolio); - dst_addr = inline_data_addr(inode, ifolio); + src_addr = inline_data_addr(inode, entry); + dst_addr = inline_data_addr(inode, ientry); memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode)); set_inode_flag(inode, FI_INLINE_DATA); set_inode_flag(inode, FI_DATA_EXIST); - folio_mark_dirty(ifolio); - f2fs_folio_put(ifolio, true); + f2fs_mark_cache_dirty(ientry); + f2fs_put_cache(ientry, true); return 1; } if (f2fs_has_inline_data(inode)) { - struct folio *ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); - f2fs_truncate_inline_inode(inode, ifolio, 0); + struct f2fs_cached_block *ientry = f2fs_get_inode_cache(sbi, inode->i_ino); + + if (IS_ERR(ientry)) + return PTR_ERR(ientry); + f2fs_truncate_inline_inode(inode, ientry, 0); stat_dec_inline_inode(inode); clear_inode_flag(inode, FI_INLINE_DATA); - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) { int ret; @@ -355,50 +357,50 @@ int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio) struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir, const struct f2fs_filename *fname, - struct folio **res_folio, + void **dentry_block, bool use_hash) { struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb); struct f2fs_dir_entry *de; struct f2fs_dentry_ptr d; - struct folio *ifolio; + struct f2fs_cached_block *ientry; void *inline_dentry; - ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); - if (IS_ERR(ifolio)) { - *res_folio = ifolio; + ientry = f2fs_get_inode_cache(sbi, dir->i_ino); + if (IS_ERR(ientry)) { + *dentry_block = ientry; return NULL; } - inline_dentry = inline_data_addr(dir, ifolio); + inline_dentry = inline_data_addr(dir, ientry); make_dentry_ptr_inline(dir, &d, inline_dentry); de = f2fs_find_target_dentry(&d, fname, NULL, use_hash); - folio_unlock(ifolio); + f2fs_unlock_cache(ientry); if (IS_ERR(de)) { - *res_folio = ERR_CAST(de); + *dentry_block = ERR_CAST(de); de = NULL; } if (de) - *res_folio = ifolio; + *dentry_block = f2fs_cache_make_dentry_block(ientry); else - f2fs_folio_put(ifolio, false); + f2fs_put_cache(ientry, false); return de; } int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent, - struct folio *ifolio) + struct f2fs_cached_block *ientry) { struct f2fs_dentry_ptr d; void *inline_dentry; - inline_dentry = inline_data_addr(inode, ifolio); + inline_dentry = inline_data_addr(inode, ientry); make_dentry_ptr_inline(inode, &d, inline_dentry); f2fs_do_make_empty_dir(inode, parent, &d); - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); /* update i_size to MAX_INLINE_DATA */ if (i_size_read(inode) < MAX_INLINE_DATA(inode)) @@ -410,22 +412,23 @@ int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent, * NOTE: ipage is grabbed by caller, but if any error occurs, we should * release ipage in this function. */ -static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, - void *inline_dentry) +static int f2fs_move_inline_dirents(struct inode *dir, + struct f2fs_cached_block *ientry, + void *inline_dentry) { struct folio *folio; struct dnode_of_data dn; - struct f2fs_dentry_block *dentry_blk; + void *dentry_blk; struct f2fs_dentry_ptr src, dst; int err; folio = f2fs_grab_cache_folio(dir->i_mapping, 0, true); if (IS_ERR(folio)) { - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return PTR_ERR(folio); } - set_new_dnode(&dn, dir, ifolio, NULL, 0); + set_new_dnode(&dn, dir, ientry, NULL, 0); err = f2fs_reserve_block(&dn, 0); if (err) goto out; @@ -441,7 +444,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, goto out; } - f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_folio_wait_writeback(folio, true, true); dentry_blk = folio_address(folio); @@ -449,7 +452,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, * Start by zeroing the full block, to ensure that all unused space is * zeroed and no uninitialized memory is leaked to disk. */ - memset(dentry_blk, 0, F2FS_BLKSIZE); + memset(dentry_blk, 0, F2FS_BLKSIZE(F2FS_I_SB(dir))); make_dentry_ptr_inline(dir, &src, inline_dentry); make_dentry_ptr_block(dir, &dst, dentry_blk); @@ -464,7 +467,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, folio_mark_dirty(folio); /* clear inline dir and flag after data writeback */ - f2fs_truncate_inline_inode(dir, ifolio, 0); + f2fs_truncate_inline_inode(dir, ientry, 0); stat_dec_inline_dir(dir); clear_inode_flag(dir, FI_INLINE_DENTRY); @@ -478,8 +481,8 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, F2FS_I(dir)->i_inline_xattr_size = 0; f2fs_i_depth_write(dir, 1); - if (i_size_read(dir) < PAGE_SIZE) - f2fs_i_size_write(dir, PAGE_SIZE); + if (i_size_read(dir) < F2FS_BLKSIZE(F2FS_I_SB(dir))) + f2fs_i_size_write(dir, F2FS_BLKSIZE(F2FS_I_SB(dir))); out: f2fs_folio_put(folio, true); return err; @@ -544,8 +547,8 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) return err; } -static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio, - void *inline_dentry) +static int f2fs_move_rehashed_dirents(struct inode *dir, + struct f2fs_cached_block *ientry, void *inline_dentry) { void *backup_dentry; int err; @@ -553,20 +556,20 @@ static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio, backup_dentry = f2fs_kmalloc(F2FS_I_SB(dir), MAX_INLINE_DATA(dir), GFP_F2FS_ZERO); if (!backup_dentry) { - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return -ENOMEM; } memcpy(backup_dentry, inline_dentry, MAX_INLINE_DATA(dir)); - f2fs_truncate_inline_inode(dir, ifolio, 0); + f2fs_truncate_inline_inode(dir, ientry, 0); - folio_unlock(ifolio); + f2fs_unlock_cache(ientry); err = f2fs_add_inline_entries(dir, backup_dentry); if (err) goto recover; - folio_lock(ifolio); + f2fs_lock_cache(ientry); stat_dec_inline_dir(dir); clear_inode_flag(dir, FI_INLINE_DENTRY); @@ -582,31 +585,31 @@ static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio, kfree(backup_dentry); return 0; recover: - folio_lock(ifolio); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_lock_cache(ientry); + f2fs_cache_wait_writeback(ientry); memcpy(inline_dentry, backup_dentry, MAX_INLINE_DATA(dir)); f2fs_i_depth_write(dir, 0); f2fs_i_size_write(dir, MAX_INLINE_DATA(dir)); - folio_mark_dirty(ifolio); - f2fs_folio_put(ifolio, true); + f2fs_mark_cache_dirty(ientry); + f2fs_put_cache(ientry, true); kfree(backup_dentry); return err; } -static int do_convert_inline_dir(struct inode *dir, struct folio *ifolio, - void *inline_dentry) +static int do_convert_inline_dir(struct inode *dir, + struct f2fs_cached_block *ientry, void *inline_dentry) { if (!F2FS_I(dir)->i_dir_level) - return f2fs_move_inline_dirents(dir, ifolio, inline_dentry); + return f2fs_move_inline_dirents(dir, ientry, inline_dentry); else - return f2fs_move_rehashed_dirents(dir, ifolio, inline_dentry); + return f2fs_move_rehashed_dirents(dir, ientry, inline_dentry); } int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); - struct folio *ifolio; + struct f2fs_cached_block *ientry; struct f2fs_filename fname; struct f2fs_lock_context lc; void *inline_dentry = NULL; @@ -621,22 +624,22 @@ int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry) if (err) goto out; - ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); - if (IS_ERR(ifolio)) { - err = PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, dir->i_ino); + if (IS_ERR(ientry)) { + err = PTR_ERR(ientry); goto out_fname; } - if (f2fs_has_enough_room(dir, ifolio, &fname)) { - f2fs_folio_put(ifolio, true); + if (f2fs_has_enough_room(dir, ientry, &fname)) { + f2fs_put_cache(ientry, true); goto out_fname; } - inline_dentry = inline_data_addr(dir, ifolio); + inline_dentry = inline_data_addr(dir, ientry); - err = do_convert_inline_dir(dir, ifolio, inline_dentry); + err = do_convert_inline_dir(dir, ientry, inline_dentry); if (!err) - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); out_fname: f2fs_free_filename(&fname); out: @@ -648,24 +651,24 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, struct inode *inode, nid_t ino, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); - struct folio *ifolio; + struct f2fs_cached_block *ientry; unsigned int bit_pos; void *inline_dentry = NULL; struct f2fs_dentry_ptr d; int slots = GET_DENTRY_SLOTS(fname->disk_name.len); - struct folio *folio = NULL; + struct f2fs_cached_block *nentry = NULL; int err = 0; - ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(sbi, dir->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); - inline_dentry = inline_data_addr(dir, ifolio); + inline_dentry = inline_data_addr(dir, ientry); make_dentry_ptr_inline(dir, &d, inline_dentry); bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max); if (bit_pos >= d.max) { - err = do_convert_inline_dir(dir, ifolio, inline_dentry); + err = do_convert_inline_dir(dir, ientry, inline_dentry); if (err) return err; err = -EAGAIN; @@ -675,19 +678,19 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, if (inode) { f2fs_down_write_nested(&F2FS_I(inode)->i_sem, SINGLE_DEPTH_NESTING); - folio = f2fs_init_inode_metadata(inode, dir, fname, ifolio); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + nentry = f2fs_init_inode_metadata(inode, dir, fname, ientry); + if (IS_ERR(nentry)) { + err = PTR_ERR(nentry); goto fail; } } - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash, bit_pos); - folio_mark_dirty(ifolio); + f2fs_mark_cache_dirty(ientry); /* we don't need to mark_inode_dirty now */ if (inode) { @@ -695,9 +698,9 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, /* synchronize inode page's data from inode cache */ if (is_inode_flag_set(inode, FI_NEW_INODE)) - f2fs_update_inode(inode, folio); + f2fs_update_inode(inode, nentry); - f2fs_folio_put(folio, true); + f2fs_put_cache(nentry, true); } f2fs_update_parent_metadata(dir, inode, 0); @@ -705,12 +708,12 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, if (inode) f2fs_up_write(&F2FS_I(inode)->i_sem); out: - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return err; } void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, - struct folio *folio, struct inode *dir, struct inode *inode) + struct f2fs_cached_block *ientry, struct inode *dir, struct inode *inode) { struct f2fs_dentry_ptr d; void *inline_dentry; @@ -718,18 +721,18 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, unsigned int bit_pos; int i; - folio_lock(folio); - f2fs_folio_wait_writeback(folio, NODE, true, true); + f2fs_lock_cache(ientry); + f2fs_cache_wait_writeback(ientry); - inline_dentry = inline_data_addr(dir, folio); + inline_dentry = inline_data_addr(dir, ientry); make_dentry_ptr_inline(dir, &d, inline_dentry); bit_pos = dentry - d.dentry; for (i = 0; i < slots; i++) __clear_bit_le(bit_pos + i, d.bitmap); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + f2fs_mark_cache_dirty(ientry); + f2fs_put_cache(ientry, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); f2fs_mark_inode_dirty_sync(dir, true); @@ -741,21 +744,21 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, bool f2fs_empty_inline_dir(struct inode *dir) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); - struct folio *ifolio; + struct f2fs_cached_block *ientry; unsigned int bit_pos = 2; void *inline_dentry; struct f2fs_dentry_ptr d; - ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); - if (IS_ERR(ifolio)) + ientry = f2fs_get_inode_cache(sbi, dir->i_ino); + if (IS_ERR(ientry)) return false; - inline_dentry = inline_data_addr(dir, ifolio); + inline_dentry = inline_data_addr(dir, ientry); make_dentry_ptr_inline(dir, &d, inline_dentry); bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos); - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); if (bit_pos < d.max) return false; @@ -767,7 +770,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, struct fscrypt_str *fstr) { struct inode *inode = file_inode(file); - struct folio *ifolio = NULL; + struct f2fs_cached_block *ientry = NULL; struct f2fs_dentry_ptr d; void *inline_dentry = NULL; int err; @@ -777,17 +780,17 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, if (ctx->pos == d.max) return 0; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); /* * f2fs_readdir was protected by inode.i_rwsem, it is safe to access * ipage without page's lock held. */ - folio_unlock(ifolio); + f2fs_unlock_cache(ientry); - inline_dentry = inline_data_addr(inode, ifolio); + inline_dentry = inline_data_addr(inode, ientry); make_dentry_ptr_inline(inode, &d, inline_dentry); @@ -795,7 +798,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, if (!err) ctx->pos = d.max; - f2fs_folio_put(ifolio, false); + f2fs_put_cache(ientry, false); return err < 0 ? err : 0; } @@ -806,12 +809,12 @@ int f2fs_inline_data_fiemap(struct inode *inode, __u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_LAST; struct node_info ni; - struct folio *ifolio; + struct f2fs_cached_block *ientry; int err = 0; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); if ((S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) && !f2fs_has_inline_data(inode)) { @@ -825,13 +828,13 @@ int f2fs_inline_data_fiemap(struct inode *inode, } if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { - err = f2fs_write_single_node_folio(ifolio, true, false, FS_NODE_IO); + err = f2fs_write_node_cache(ientry, true, false, FS_NODE_IO); if (err) return err; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); + f2fs_cache_wait_writeback(ientry); } ilen = min_t(size_t, MAX_INLINE_DATA(inode), i_size_read(inode)); if (start >= ilen) @@ -846,8 +849,8 @@ int f2fs_inline_data_fiemap(struct inode *inode, if (__is_valid_data_blkaddr(ni.blk_addr)) { byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits; - byteaddr += (char *)inline_data_addr(inode, ifolio) - - (char *)F2FS_INODE(ifolio); + byteaddr += (char *)inline_data_addr(inode, ientry) - + (char *)F2FS_INODE(ientry); } else { f2fs_bug_on(F2FS_I_SB(inode), ni.blk_addr != NEW_ADDR); flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN; @@ -855,6 +858,6 @@ int f2fs_inline_data_fiemap(struct inode *inode, err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags); trace_f2fs_fiemap(inode, start, byteaddr, ilen, flags, err); out: - f2fs_folio_put(ifolio, true); + f2fs_put_cache(ientry, true); return err; } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 96cc0e777567..9164a2b5d9f0 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -82,9 +82,9 @@ void f2fs_set_inode_flags(struct inode *inode) S_ENCRYPTED|S_VERITY|S_CASEFOLD); } -static void __get_inode_rdev(struct inode *inode, struct folio *node_folio) +static void __get_inode_rdev(struct inode *inode, struct f2fs_cached_block *ientry) { - __le32 *addr = get_dnode_addr(inode, node_folio); + __le32 *addr = get_dnode_addr(inode, ientry); if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { @@ -95,9 +95,9 @@ static void __get_inode_rdev(struct inode *inode, struct folio *node_folio) } } -static void __set_inode_rdev(struct inode *inode, struct folio *node_folio) +static void __set_inode_rdev(struct inode *inode, struct f2fs_cached_block *ientry) { - __le32 *addr = get_dnode_addr(inode, node_folio); + __le32 *addr = get_dnode_addr(inode, ientry); if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { if (old_valid_dev(inode->i_rdev)) { @@ -111,34 +111,33 @@ static void __set_inode_rdev(struct inode *inode, struct folio *node_folio) } } -static void __recover_inline_status(struct inode *inode, struct folio *ifolio) +static void __recover_inline_status(struct inode *inode, struct f2fs_cached_block *ientry) { - void *inline_data = inline_data_addr(inode, ifolio); + void *inline_data = inline_data_addr(inode, ientry); __le32 *start = inline_data; __le32 *end = start + MAX_INLINE_DATA(inode) / sizeof(__le32); while (start < end) { if (*start++) { - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); set_inode_flag(inode, FI_DATA_EXIST); - set_raw_inline(inode, F2FS_INODE(ifolio)); - folio_mark_dirty(ifolio); + set_raw_inline(inode, F2FS_INODE(ientry)); + f2fs_mark_cache_dirty(ientry); return; } } return; } -static -bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) +static bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry) { - struct f2fs_inode *ri = &F2FS_NODE(folio)->i; + struct f2fs_inode *ri = F2FS_INODE(entry); if (!f2fs_sb_has_inode_chksum(sbi)) return false; - if (!IS_INODE(folio) || !(ri->i_inline & F2FS_EXTRA_ATTR)) + if (!IS_INODE(sbi, entry) || !(ri->i_inline & F2FS_EXTRA_ATTR)) return false; if (!F2FS_FITS_IN_INODE(ri, le16_to_cpu(ri->i_extra_isize), @@ -148,11 +147,10 @@ bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) return true; } -static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) +static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry) { - struct f2fs_node *node = F2FS_NODE(folio); - struct f2fs_inode *ri = &node->i; - __le32 ino = node->footer.ino; + struct f2fs_inode *ri = F2FS_INODE(entry); + __le32 ino = F2FS_NODE_FOOTER(sbi, entry)->ino; __le32 gen = ri->i_generation; __u32 chksum, chksum_seed; __u32 dummy_cs = 0; @@ -166,11 +164,11 @@ static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) chksum = f2fs_chksum(chksum, (__u8 *)&dummy_cs, cs_size); offset += cs_size; chksum = f2fs_chksum(chksum, (__u8 *)ri + offset, - F2FS_BLKSIZE - offset); + F2FS_BLKSIZE(sbi) - offset); return chksum; } -bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio) +bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry) { struct f2fs_inode *ri; __u32 provided, calculated; @@ -178,35 +176,33 @@ bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio) if (unlikely(is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))) return true; -#ifdef CONFIG_F2FS_CHECK_FS - if (!f2fs_enable_inode_chksum(sbi, folio)) -#else - if (!f2fs_enable_inode_chksum(sbi, folio) || - folio_test_dirty(folio) || - folio_test_writeback(folio)) -#endif + if (!f2fs_enable_inode_chksum(sbi, entry)) return true; +#ifndef CONFIG_F2FS_CHECK_FS + if (f2fs_cache_test_dirty(entry) || f2fs_cache_test_writeback(entry)) + return true; +#endif - ri = &F2FS_NODE(folio)->i; + ri = F2FS_INODE(entry); provided = le32_to_cpu(ri->i_inode_checksum); - calculated = f2fs_inode_chksum(sbi, folio); + calculated = f2fs_inode_chksum(sbi, entry); if (provided != calculated) - f2fs_warn(sbi, "checksum invalid, nid = %lu, ino_of_node = %x, %x vs. %x", - folio->index, ino_of_node(folio), + f2fs_warn(sbi, "checksum invalid, nid = %lu, ino_of_node = %u, %x vs. %x", + entry->index, ino_of_node(sbi, entry), provided, calculated); return provided == calculated; } -void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio) +void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry) { - struct f2fs_inode *ri = &F2FS_NODE(folio)->i; + struct f2fs_inode *ri = F2FS_INODE(entry); - if (!f2fs_enable_inode_chksum(sbi, folio)) + if (!f2fs_enable_inode_chksum(sbi, entry)) return; - ri->i_inode_checksum = cpu_to_le32(f2fs_inode_chksum(sbi, folio)); + ri->i_inode_checksum = cpu_to_le32(f2fs_inode_chksum(sbi, entry)); } static bool sanity_check_compress_inode(struct inode *inode, @@ -222,11 +218,11 @@ static bool sanity_check_compress_inode(struct inode *inode, return false; } if (le64_to_cpu(ri->i_compr_blocks) > - SECTOR_TO_BLOCK(inode->i_blocks)) { + SECTOR_TO_BLOCK(sbi, inode->i_blocks)) { f2fs_warn(sbi, "%s: inode (ino=%llx) has inconsistent i_compr_blocks:%llu, i_blocks:%llu, run fsck to fix", __func__, inode->i_ino, le64_to_cpu(ri->i_compr_blocks), - SECTOR_TO_BLOCK(inode->i_blocks)); + SECTOR_TO_BLOCK(sbi, inode->i_blocks)); return false; } if (ri->i_log_cluster_size < MIN_COMPRESS_LOG_SIZE || @@ -281,28 +277,29 @@ static bool sanity_check_compress_inode(struct inode *inode, return false; } -static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) +static bool sanity_check_inode(struct inode *inode, + struct f2fs_cached_block *node_entry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); - struct f2fs_inode *ri = F2FS_INODE(node_folio); + struct f2fs_inode *ri = F2FS_INODE(node_entry); unsigned long long iblocks; - iblocks = le64_to_cpu(F2FS_INODE(node_folio)->i_blocks); + iblocks = le64_to_cpu(ri->i_blocks); if (!iblocks) { f2fs_warn(sbi, "%s: corrupted inode i_blocks i_ino=%llx iblocks=%llu, run fsck to fix.", __func__, inode->i_ino, iblocks); return false; } - if (ino_of_node(node_folio) != nid_of_node(node_folio)) { + if (ino_of_node(sbi, node_entry) != nid_of_node(sbi, node_entry)) { f2fs_warn(sbi, "%s: corrupted inode footer i_ino=%llx, ino,nid: [%u, %u] run fsck to fix.", __func__, inode->i_ino, - ino_of_node(node_folio), nid_of_node(node_folio)); + ino_of_node(sbi, node_entry), nid_of_node(sbi, node_entry)); return false; } - if (ino_of_node(node_folio) == fi->i_xattr_nid) { + if (ino_of_node(sbi, node_entry) == fi->i_xattr_nid) { f2fs_warn(sbi, "%s: corrupted inode i_ino=%llx, xnid=%x, run fsck to fix.", __func__, inode->i_ino, fi->i_xattr_nid); return false; @@ -338,12 +335,13 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) } if (f2fs_sb_has_flexible_inline_xattr(sbi) && - (fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE || + (fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE(i_blocksize(inode)) || (f2fs_has_inline_xattr(inode) && fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE))) { - f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu", + f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %zu", __func__, inode->i_ino, fi->i_inline_xattr_size, - MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE); + MIN_INLINE_XATTR_SIZE, + (size_t)MAX_INLINE_XATTR_SIZE(i_blocksize(inode))); return false; } @@ -375,7 +373,7 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) } } - if (f2fs_sanity_check_inline_data(inode, node_folio)) { + if (f2fs_sanity_check_inline_data(inode, node_entry)) { f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_data, run fsck to fix", __func__, inode->i_ino, inode->i_mode); return false; @@ -428,7 +426,7 @@ static int do_read_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); - struct folio *node_folio; + struct f2fs_cached_block *node_entry; struct f2fs_inode *ri; projid_t i_projid; @@ -436,18 +434,19 @@ static int do_read_inode(struct inode *inode) if (f2fs_check_nid_range(sbi, inode->i_ino)) return -EINVAL; - node_folio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(node_folio)) - return PTR_ERR(node_folio); + node_entry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(node_entry)) + return PTR_ERR(node_entry); - ri = F2FS_INODE(node_folio); + ri = F2FS_INODE(node_entry); inode->i_mode = le16_to_cpu(ri->i_mode); i_uid_write(inode, le32_to_cpu(ri->i_uid)); i_gid_write(inode, le32_to_cpu(ri->i_gid)); set_nlink(inode, le32_to_cpu(ri->i_links)); inode->i_size = le64_to_cpu(ri->i_size); - inode->i_blocks = SECTOR_FROM_BLOCK(le64_to_cpu(ri->i_blocks) - 1); + inode->i_blocks = SECTOR_FROM_BLOCK(sbi, + le64_to_cpu(ri->i_blocks) - 1); inode_set_atime(inode, le64_to_cpu(ri->i_atime), le32_to_cpu(ri->i_atime_nsec)); @@ -490,8 +489,8 @@ static int do_read_inode(struct inode *inode) fi->i_inline_xattr_size = 0; } - if (!sanity_check_inode(inode, node_folio)) { - f2fs_folio_put(node_folio, true); + if (!sanity_check_inode(inode, node_entry)) { + f2fs_put_cache(node_entry, true); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); @@ -500,17 +499,17 @@ static int do_read_inode(struct inode *inode) /* check data exist */ if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode)) - __recover_inline_status(inode, node_folio); + __recover_inline_status(inode, node_entry); /* try to recover cold bit for non-dir inode */ - if (!S_ISDIR(inode->i_mode) && !is_cold_node(node_folio)) { - f2fs_folio_wait_writeback(node_folio, NODE, true, true); - set_cold_node(node_folio, false); - folio_mark_dirty(node_folio); + if (!S_ISDIR(inode->i_mode) && !is_cold_node(sbi, node_entry)) { + f2fs_cache_wait_writeback(node_entry); + set_cold_node(sbi, node_entry, false); + f2fs_mark_cache_dirty(node_entry); } /* get rdev by using inline_info */ - __get_inode_rdev(inode, node_folio); + __get_inode_rdev(inode, node_entry); if (!f2fs_need_inode_block_update(sbi, inode->i_ino)) fi->last_disk_size = inode->i_size; @@ -553,18 +552,18 @@ static int do_read_inode(struct inode *inode) init_idisk_time(inode); - if (!sanity_check_extent_cache(inode, node_folio)) { - f2fs_folio_put(node_folio, true); + if (!sanity_check_extent_cache(inode, node_entry)) { + f2fs_put_cache(node_entry, true); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); return -EFSCORRUPTED; } /* Need all the flag bits */ - f2fs_init_read_extent_tree(inode, node_folio); + f2fs_init_read_extent_tree(inode, node_entry); f2fs_init_age_extent_tree(inode); - f2fs_folio_put(node_folio, true); + f2fs_put_cache(node_entry, true); stat_inc_inline_xattr(inode); stat_inc_inline_inode(inode); @@ -575,17 +574,6 @@ static int do_read_inode(struct inode *inode) return 0; } -static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino) -{ - if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) - return true; -#ifdef CONFIG_F2FS_FS_COMPRESSION - if (test_opt(sbi, COMPRESS_CACHE) && ino == F2FS_COMPRESS_INO(sbi)) - return true; -#endif - return false; -} - struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -597,48 +585,17 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) return ERR_PTR(-ENOMEM); if (!(inode_state_read_once(inode) & I_NEW)) { - if (is_meta_ino(sbi, ino)) { - f2fs_err(sbi, "inaccessible inode: %lu, run fsck to repair", ino); - set_sbi_flag(sbi, SBI_NEED_FSCK); - ret = -EFSCORRUPTED; - trace_f2fs_iget_exit(inode, ret); - iput(inode); - f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); - fserror_report_file_metadata(inode, ret, GFP_NOFS); - return ERR_PTR(ret); - } - trace_f2fs_iget(inode); return inode; } - if (is_meta_ino(sbi, ino)) - goto make_now; - ret = do_read_inode(inode); if (ret) goto bad_inode; -make_now: + f2fs_set_inode_flags(inode); - if (ino == F2FS_NODE_INO(sbi)) { - inode->i_mapping->a_ops = &f2fs_node_aops; - mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); - } else if (ino == F2FS_META_INO(sbi)) { - inode->i_mapping->a_ops = &f2fs_meta_aops; - mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); - } else if (ino == F2FS_COMPRESS_INO(sbi)) { -#ifdef CONFIG_F2FS_FS_COMPRESSION - inode->i_mapping->a_ops = &f2fs_compress_aops; - /* - * generic_error_remove_folio only truncates pages of regular - * inode - */ - inode->i_mode |= S_IFREG; -#endif - mapping_set_gfp_mask(inode->i_mapping, - GFP_NOFS | __GFP_HIGHMEM | __GFP_MOVABLE); - } else if (S_ISREG(inode->i_mode)) { + if (S_ISREG(inode->i_mode)) { inode->i_op = &f2fs_file_inode_operations; inode->i_fop = &f2fs_file_operations; inode->i_mapping->a_ops = &f2fs_dblock_aops; @@ -694,25 +651,28 @@ struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino) return inode; } -void f2fs_update_inode(struct inode *inode, struct folio *node_folio) +void f2fs_update_inode(struct inode *inode, + struct f2fs_cached_block *node_entry) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_inode *ri; struct extent_tree *et = fi->extent_tree[EX_READ]; - f2fs_folio_wait_writeback(node_folio, NODE, true, true); - folio_mark_dirty(node_folio); + f2fs_cache_wait_writeback(node_entry); + f2fs_mark_cache_dirty(node_entry); f2fs_inode_synced(inode); - ri = F2FS_INODE(node_folio); + ri = F2FS_INODE(node_entry); ri->i_mode = cpu_to_le16(inode->i_mode); ri->i_advise = fi->i_advise; ri->i_uid = cpu_to_le32(i_uid_read(inode)); ri->i_gid = cpu_to_le32(i_gid_read(inode)); ri->i_links = cpu_to_le32(inode->i_nlink); - ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(READ_ONCE(inode->i_blocks)) + 1); + ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(sbi, + READ_ONCE(inode->i_blocks)) + 1); if (!f2fs_is_atomic_file(inode) || is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) @@ -780,27 +740,27 @@ void f2fs_update_inode(struct inode *inode, struct folio *node_folio) } } - __set_inode_rdev(inode, node_folio); + __set_inode_rdev(inode, node_entry); /* deleted inode */ if (inode->i_nlink == 0) - folio_clear_f2fs_inline(node_folio); + f2fs_cache_clear_inline(node_entry); init_idisk_time(inode); #ifdef CONFIG_F2FS_CHECK_FS - f2fs_inode_chksum_set(F2FS_I_SB(inode), node_folio); + f2fs_inode_chksum_set(F2FS_I_SB(inode), node_entry); #endif } -void f2fs_update_inode_page(struct inode *inode) +void f2fs_update_inode_cache(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct folio *node_folio; + struct f2fs_cached_block *node_entry; int count = 0; retry: - node_folio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(node_folio)) { - int err = PTR_ERR(node_folio); + node_entry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(node_entry)) { + int err = PTR_ERR(node_entry); /* The node block was truncated. */ if (err == -ENOENT) @@ -816,18 +776,14 @@ void f2fs_update_inode_page(struct inode *inode) f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE); return; } - f2fs_update_inode(inode, node_folio); - f2fs_folio_put(node_folio, true); + f2fs_update_inode(inode, node_entry); + f2fs_put_cache(node_entry, true); } int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - if (inode->i_ino == F2FS_NODE_INO(sbi) || - inode->i_ino == F2FS_META_INO(sbi)) - return 0; - /* * atime could be updated without dirtying f2fs inode in lazytime mode */ @@ -851,7 +807,7 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) * We need to balance fs here to prevent from producing dirty node pages * during the urgent cleaning time when running out of free sections. */ - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); if (wbc && wbc->nr_to_write) f2fs_balance_fs(sbi, true); return 0; @@ -892,7 +848,7 @@ static void f2fs_evict_inode_work(struct work_struct *work) /* * Return true, if we shouldn't go through post_evict_inode. */ -static bool f2fs_pre_evict_inode(struct inode *inode) +static void f2fs_pre_evict_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); @@ -918,19 +874,12 @@ static bool f2fs_pre_evict_inode(struct inode *inode) test_opt(sbi, COMPRESS_CACHE) && f2fs_compressed_file(inode)) f2fs_invalidate_compress_pages(sbi, inode->i_ino); - if (inode->i_ino == F2FS_NODE_INO(sbi) || - inode->i_ino == F2FS_META_INO(sbi) || - inode->i_ino == F2FS_COMPRESS_INO(sbi)) - return true; - f2fs_bug_on(sbi, get_dirty_pages(inode)); f2fs_remove_dirty_inode(inode); f2fs_remove_donate_inode(inode); if (!IS_DEVICE_ALIASING(inode)) f2fs_destroy_extent_tree(inode); - - return false; } static void f2fs_delete_inode(struct inode *inode) @@ -964,7 +913,7 @@ static void f2fs_delete_inode(struct inode *inode) goto error_check; f2fs_lock_op(sbi, &lc); - err = f2fs_remove_inode_page(inode); + err = f2fs_remove_inode_cache(inode); f2fs_unlock_op(sbi, &lc); if (err == -ENOENT) { @@ -996,13 +945,13 @@ static void f2fs_delete_inode(struct inode *inode) if (!err) goto unfreeze_out; - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); if (dquot_initialize_needed(inode)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); /* - * If both f2fs_truncate() and f2fs_update_inode_page() failed + * If both f2fs_truncate() and f2fs_update_inode_cache() failed * due to fuzzed corrupted inode, call f2fs_inode_synced() to * avoid triggering later f2fs_bug_on(). */ @@ -1046,18 +995,17 @@ static void f2fs_post_evict_inode(struct inode *inode) /* for the case f2fs_new_inode() was failed, .i_ino is zero, skip it */ if (inode->i_ino) - invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, - inode->i_ino); + f2fs_invalidate_node_cache(sbi, inode->i_ino); if (xnid) - invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); + f2fs_invalidate_node_cache(sbi, xnid); if (!inode->i_nlink) goto skip_record; if (is_inode_flag_set(inode, FI_APPEND_WRITE)) - record_bits = BIT(APPEND_INO); + record_bits |= BIT(APPEND_INO); if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) - record_bits = BIT(UPDATE_INO); + record_bits |= BIT(UPDATE_INO); if (!record_bits) goto skip_record; @@ -1093,15 +1041,13 @@ static void f2fs_post_evict_inode(struct inode *inode) */ void f2fs_evict_inode(struct inode *inode) { - if (f2fs_pre_evict_inode(inode)) - goto clear_out; + f2fs_pre_evict_inode(inode); if (!inode->i_nlink && !is_bad_inode(inode)) f2fs_delete_inode(inode); f2fs_post_evict_inode(inode); -clear_out: fscrypt_put_encryption_info(inode); clear_inode(inode); } @@ -1124,7 +1070,7 @@ void f2fs_handle_failed_inode(struct inode *inode, * we must call this to avoid inode being remained as dirty, resulting * in a panic when flushing dirty inodes in gdirty_list. */ - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); f2fs_inode_synced(inode); /* don't make bad inode, since it becomes a regular file. */ diff --git a/fs/f2fs/iostat.h b/fs/f2fs/iostat.h index 2025225b5bed..61c6bc8e3119 100644 --- a/fs/f2fs/iostat.h +++ b/fs/f2fs/iostat.h @@ -60,6 +60,13 @@ static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio) return iostat_ctx->post_read_ctx; } +static inline void iostat_set_post_read_ctx(struct bio *bio, void *ctx) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + + iostat_ctx->post_read_ctx = ctx; +} + extern void iostat_update_and_unbind_ctx(struct bio *bio); extern void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, struct bio *bio, struct bio_post_read_ctx *ctx); @@ -81,6 +88,10 @@ static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio) { return bio->bi_private; } +static inline void iostat_set_post_read_ctx(struct bio *bio, void *ctx) +{ + bio->bi_private = ctx; +} static inline int f2fs_init_iostat_processing(void) { return 0; } static inline void f2fs_destroy_iostat_processing(void) {} static inline int f2fs_init_iostat(struct f2fs_sb_info *sbi) { return 0; } diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 233c07fec171..ce5d536d892b 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -473,12 +473,13 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *f2fs_get_parent(struct dentry *child) { - struct folio *folio; - unsigned long ino = f2fs_inode_by_name(d_inode(child), &dotdot_name, &folio); + void *dentry_block = NULL; + unsigned long ino = f2fs_inode_by_name(d_inode(child), + &dotdot_name, &dentry_block); if (!ino) { - if (IS_ERR(folio)) - return ERR_CAST(folio); + if (IS_ERR(dentry_block)) + return ERR_CAST(dentry_block); return ERR_PTR(-ENOENT); } return d_obtain_alias(f2fs_iget(child->d_sb, ino)); @@ -489,7 +490,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, { struct inode *inode = NULL; struct f2fs_dir_entry *de; - struct folio *folio; + void *dentry_block = NULL; struct dentry *new; nid_t ino = -1; int err = 0; @@ -507,12 +508,12 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, goto out_splice; if (err) goto out; - de = __f2fs_find_entry(dir, &fname, &folio); + de = __f2fs_find_entry(dir, &fname, &dentry_block); f2fs_free_filename(&fname); if (!de) { - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + if (IS_ERR(dentry_block)) { + err = PTR_ERR(dentry_block); goto out; } err = -ENOENT; @@ -520,7 +521,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, } ino = le32_to_cpu(de->ino); - f2fs_folio_put(folio, false); + f2fs_put_dentry_block(dentry_block, false); inode = f2fs_iget(dir->i_sb, ino); if (IS_ERR(inode)) { @@ -572,7 +573,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode, struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct f2fs_dir_entry *de; struct f2fs_lock_context lc; - struct folio *folio; + void *dentry_block = NULL; int err; if (IS_DEVICE_ALIASING(inode)) @@ -588,9 +589,9 @@ static int __do_unlink(struct inode *dir, struct inode *inode, if (err) return err; - de = f2fs_find_entry(dir, name, &folio); + de = f2fs_find_entry(dir, name, &dentry_block); if (!de) - return IS_ERR(folio) ? PTR_ERR(folio) : 0; + return IS_ERR(dentry_block) ? PTR_ERR(dentry_block) : 0; if (unlikely(inode->i_nlink == 0)) { f2fs_warn(sbi, "%s: inode (ino=%llx) has zero i_nlink", @@ -610,7 +611,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode, f2fs_unlock_op(sbi, &lc); goto err_out; } - f2fs_delete_entry(de, folio, dir, inode); + f2fs_delete_entry(de, dentry_block, dir, inode); f2fs_unlock_op(sbi, &lc); return 0; @@ -618,7 +619,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode, err = -EFSCORRUPTED; set_sbi_flag(sbi, SBI_NEED_FSCK); err_out: - f2fs_folio_put(folio, false); + f2fs_put_dentry_block(dentry_block, false); return err; } @@ -967,8 +968,9 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); struct inode *whiteout = NULL; - struct folio *old_dir_folio = NULL; - struct folio *old_folio, *new_folio = NULL; + void *old_dir_block = NULL; + void *old_block = NULL; + void *new_block = NULL; struct f2fs_dir_entry *old_dir_entry = NULL; struct f2fs_dir_entry *old_entry; struct f2fs_dir_entry *new_entry; @@ -1032,18 +1034,18 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, } err = -ENOENT; - old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_block); if (!old_entry) { - if (IS_ERR(old_folio)) - err = PTR_ERR(old_folio); + if (IS_ERR(old_block)) + err = PTR_ERR(old_block); goto out; } if (old_is_dir && old_dir != new_dir) { - old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_folio); + old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_block); if (!old_dir_entry) { - if (IS_ERR(old_dir_folio)) - err = PTR_ERR(old_dir_folio); + if (IS_ERR(old_dir_block)) + err = PTR_ERR(old_dir_block); goto out_old; } } @@ -1060,10 +1062,10 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, err = -ENOENT; new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, - &new_folio); + &new_block); if (!new_entry) { - if (IS_ERR(new_folio)) - err = PTR_ERR(new_folio); + if (IS_ERR(new_block)) + err = PTR_ERR(new_block); goto out_dir; } @@ -1075,8 +1077,8 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, if (err) goto put_out_dir; - f2fs_set_link(new_dir, new_entry, new_folio, old_inode); - new_folio = NULL; + f2fs_set_link(new_dir, new_entry, new_block, old_inode); + new_block = NULL; inode_set_ctime_current(new_inode); f2fs_down_write(&F2FS_I(new_inode)->i_sem); @@ -1115,8 +1117,8 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, inode_set_ctime_current(old_inode); f2fs_mark_inode_dirty_sync(old_inode, true); - f2fs_delete_entry(old_entry, old_folio, old_dir, NULL); - old_folio = NULL; + f2fs_delete_entry(old_entry, old_block, old_dir, NULL); + old_block = NULL; if (whiteout) { set_inode_flag(whiteout, FI_INC_LINK); @@ -1134,7 +1136,7 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, } if (old_dir_entry) - f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir); + f2fs_set_link(old_inode, old_dir_entry, old_dir_block, new_dir); if (old_is_dir) f2fs_i_links_write(old_dir, false); @@ -1158,12 +1160,12 @@ static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir, put_out_dir: f2fs_unlock_op(sbi, &lc); - f2fs_folio_put(new_folio, false); + f2fs_put_dentry_block(new_block, false); out_dir: if (old_dir_entry) - f2fs_folio_put(old_dir_folio, false); + f2fs_put_dentry_block(old_dir_block, false); out_old: - f2fs_folio_put(old_folio, false); + f2fs_put_dentry_block(old_block, false); out: iput(whiteout); return err; @@ -1175,8 +1177,10 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); - struct folio *old_dir_folio, *new_dir_folio; - struct folio *old_folio, *new_folio; + void *old_dir_block = NULL; + void *new_dir_block = NULL; + void *old_block = NULL; + void *new_block = NULL; struct f2fs_dir_entry *old_dir_entry = NULL, *new_dir_entry = NULL; struct f2fs_dir_entry *old_entry, *new_entry; struct f2fs_lock_context lc; @@ -1208,17 +1212,17 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, goto out; err = -ENOENT; - old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_block); if (!old_entry) { - if (IS_ERR(old_folio)) - err = PTR_ERR(old_folio); + if (IS_ERR(old_block)) + err = PTR_ERR(old_block); goto out; } - new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_folio); + new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_block); if (!new_entry) { - if (IS_ERR(new_folio)) - err = PTR_ERR(new_folio); + if (IS_ERR(new_block)) + err = PTR_ERR(new_block); goto out_old; } @@ -1226,20 +1230,20 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, if (old_dir != new_dir) { if (S_ISDIR(old_inode->i_mode)) { old_dir_entry = f2fs_parent_dir(old_inode, - &old_dir_folio); + &old_dir_block); if (!old_dir_entry) { - if (IS_ERR(old_dir_folio)) - err = PTR_ERR(old_dir_folio); + if (IS_ERR(old_dir_block)) + err = PTR_ERR(old_dir_block); goto out_new; } } if (S_ISDIR(new_inode->i_mode)) { new_dir_entry = f2fs_parent_dir(new_inode, - &new_dir_folio); + &new_dir_block); if (!new_dir_entry) { - if (IS_ERR(new_dir_folio)) - err = PTR_ERR(new_dir_folio); + if (IS_ERR(new_dir_block)) + err = PTR_ERR(new_dir_block); goto out_old_dir; } } @@ -1266,14 +1270,14 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, /* update ".." directory entry info of old dentry */ if (old_dir_entry) - f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir); + f2fs_set_link(old_inode, old_dir_entry, old_dir_block, new_dir); /* update ".." directory entry info of new dentry */ if (new_dir_entry) - f2fs_set_link(new_inode, new_dir_entry, new_dir_folio, old_dir); + f2fs_set_link(new_inode, new_dir_entry, new_dir_block, old_dir); /* update directory entry info of old dir inode */ - f2fs_set_link(old_dir, old_entry, old_folio, new_inode); + f2fs_set_link(old_dir, old_entry, old_block, new_inode); f2fs_down_write(&F2FS_I(old_inode)->i_sem); if (!old_dir_entry) @@ -1292,7 +1296,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_mark_inode_dirty_sync(old_dir, true); /* update directory entry info of new dir inode */ - f2fs_set_link(new_dir, new_entry, new_folio, old_inode); + f2fs_set_link(new_dir, new_entry, new_block, old_inode); f2fs_down_write(&F2FS_I(new_inode)->i_sem); if (!new_dir_entry) @@ -1327,16 +1331,16 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, return 0; out_new_dir: if (new_dir_entry) { - f2fs_folio_put(new_dir_folio, false); + f2fs_put_dentry_block(new_dir_block, false); } out_old_dir: if (old_dir_entry) { - f2fs_folio_put(old_dir_folio, false); + f2fs_put_dentry_block(old_dir_block, false); } out_new: - f2fs_folio_put(new_folio, false); + f2fs_put_dentry_block(new_block, false); out_old: - f2fs_folio_put(old_folio, false); + f2fs_put_dentry_block(old_block, false); out: return err; } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 86c2e67e43b6..44f94000d452 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -7,12 +7,12 @@ */ #include #include -#include #include #include #include #include #include +#include #include "f2fs.h" #include "node.h" @@ -85,7 +85,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) } else if (type == DIRTY_DENTS) { if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) return false; - mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); + mem_size = get_nr_caches(sbi, F2FS_DIRTY_DENTS); res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == INO_ENTRIES) { int i; @@ -109,7 +109,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) mem_size = (atomic_read(&dcc->discard_cmd_cnt) * sizeof(struct discard_cmd)) >> PAGE_SHIFT; res = mem_size < (avail_ram * nm_i->ram_thresh / 100); - } else if (type == COMPRESS_PAGE) { + } else if (type == COMPRESS_BLOCK) { #ifdef CONFIG_F2FS_FS_COMPRESSION unsigned long free_ram = val.freeram; @@ -118,11 +118,21 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) * exceed threshold, deny caching compress page. */ res = (free_ram > avail_ram * sbi->compress_watermark / 100) && - (COMPRESS_MAPPING(sbi)->nrpages < + (COMPRESS_CACHE(sbi)->num_entries < free_ram * sbi->compress_percent / 100); #else res = false; #endif + } else if (type == META_BLOCK) { + mem_size = (META_CACHE(sbi)->num_entries * + (sizeof(struct f2fs_cached_block) + + F2FS_BLKSIZE(sbi))) >> PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else if (type == NODE_BLOCK) { + mem_size = (NODE_CACHE(sbi)->num_entries * + (sizeof(struct f2fs_cached_block) + + F2FS_BLKSIZE(sbi))) >> PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else { if (!bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) return true; @@ -130,48 +140,36 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) return res; } -static void clear_node_folio_dirty(struct folio *folio) +static struct f2fs_cached_block *get_current_nat_cache(struct f2fs_sb_info *sbi, + nid_t nid) { - if (folio_test_dirty(folio)) { - f2fs_clear_page_cache_dirty_tag(folio); - folio_clear_dirty_for_io(folio); - dec_page_count(F2FS_F_SB(folio), F2FS_DIRTY_NODES); - } - folio_clear_uptodate(folio); + return f2fs_get_meta_cache_retry(sbi, current_nat_addr(sbi, nid)); } -static struct folio *get_current_nat_folio(struct f2fs_sb_info *sbi, nid_t nid) +static struct f2fs_cached_block *get_next_nat_cache(struct f2fs_sb_info *sbi, + nid_t nid) { - return f2fs_get_meta_folio_retry(sbi, current_nat_addr(sbi, nid)); -} - -static struct folio *get_next_nat_folio(struct f2fs_sb_info *sbi, nid_t nid) -{ - struct folio *src_folio; - struct folio *dst_folio; + struct f2fs_cached_block *src_entry; + struct f2fs_cached_block *dst_entry; pgoff_t dst_off; - void *src_addr; - void *dst_addr; struct f2fs_nm_info *nm_i = NM_I(sbi); dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid)); - /* get current nat block page with lock */ - src_folio = get_current_nat_folio(sbi, nid); - if (IS_ERR(src_folio)) - return src_folio; - dst_folio = f2fs_grab_meta_folio(sbi, dst_off); - f2fs_bug_on(sbi, folio_test_dirty(src_folio)); + /* get current nat cached block with lock */ + src_entry = get_current_nat_cache(sbi, nid); + if (IS_ERR(src_entry)) + return src_entry; + dst_entry = f2fs_grab_meta_cache(sbi, dst_off); + f2fs_bug_on(sbi, f2fs_cache_test_dirty(src_entry)); - src_addr = folio_address(src_folio); - dst_addr = folio_address(dst_folio); - memcpy(dst_addr, src_addr, PAGE_SIZE); - folio_mark_dirty(dst_folio); - f2fs_folio_put(src_folio, true); + memcpy(cache_address(dst_entry), cache_address(src_entry), F2FS_BLKSIZE(sbi)); + f2fs_mark_cache_dirty(dst_entry); + f2fs_put_cache(src_entry, true); - set_to_next_nat(nm_i, nid); + set_to_next_nat(sbi, nm_i, nid); - return dst_folio; + return dst_entry; } static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi, @@ -254,10 +252,11 @@ static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e) __free_nat_entry(e); } -static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i, +static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_sb_info *sbi, + struct f2fs_nm_info *nm_i, struct nat_entry *ne) { - nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); + nid_t set = NAT_BLOCK_OFFSET(sbi, ne->ni.nid); struct nat_entry_set *head; head = radix_tree_lookup(&nm_i->nat_set_root, set); @@ -274,14 +273,15 @@ static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i, return head; } -static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i, +static void __set_nat_cache_dirty(struct f2fs_sb_info *sbi, + struct f2fs_nm_info *nm_i, struct nat_entry *ne, bool init_dirty) { struct nat_entry_set *head; bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR; if (!new_ne) - head = __grab_nat_entry_set(nm_i, ne); + head = __grab_nat_entry_set(sbi, nm_i, ne); /* * update entry_cnt in below condition: @@ -330,9 +330,11 @@ static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, start, nr); } -bool f2fs_in_warm_node_list(struct folio *folio) +bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry) { - return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio); + return f2fs_is_node_cache(entry) && + IS_DNODE(sbi, entry) && is_cold_node(sbi, entry); } void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi) @@ -344,7 +346,7 @@ void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi) } static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, - struct folio *folio) + struct f2fs_cached_block *entry) { struct fsync_node_entry *fn; unsigned long flags; @@ -353,8 +355,8 @@ static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab, GFP_NOFS, true, NULL); - folio_get(folio); - fn->folio = folio; + f2fs_cache_get(entry); + fn->entry = entry; INIT_LIST_HEAD(&fn->list); spin_lock_irqsave(&sbi->fsync_node_lock, flags); @@ -367,19 +369,19 @@ static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, return seq_id; } -void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio) +void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry) { struct fsync_node_entry *fn; unsigned long flags; spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_for_each_entry(fn, &sbi->fsync_node_list, list) { - if (fn->folio == folio) { + if (fn->entry == entry) { list_del(&fn->list); sbi->fsync_node_num--; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + f2fs_put_cache(fn->entry, false); kmem_cache_free(fsync_node_entry_slab, fn); - folio_put(folio); return; } } @@ -527,7 +529,7 @@ static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, nat_set_blkaddr(e, new_blkaddr); if (!__is_valid_data_blkaddr(new_blkaddr)) set_nat_flag(e, IS_CHECKPOINTED, false); - __set_nat_cache_dirty(nm_i, e, init_dirty); + __set_nat_cache_dirty(sbi, nm_i, e, init_dirty); /* update fsync_mark if its inode nat entry is still alive */ if (ni->nid != ni->ino) @@ -578,9 +580,9 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct f2fs_nm_info *nm_i = NM_I(sbi); struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; - nid_t start_nid = START_NID(nid); + nid_t start_nid = f2fs_start_nid(sbi, nid); struct f2fs_nat_block *nat_blk; - struct folio *folio = NULL; + struct f2fs_cached_block *entry = NULL; struct f2fs_nat_entry ne; struct nat_entry *e; pgoff_t index; @@ -631,18 +633,18 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, goto sanity_check; } - /* Fill node_info from nat page */ + /* Fill node_info from nat block */ index = current_nat_addr(sbi, nid); f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); - folio = f2fs_get_meta_folio(sbi, index); - if (IS_ERR(folio)) - return PTR_ERR(folio); + entry = f2fs_get_meta_cache(sbi, index); + if (IS_ERR(entry)) + return PTR_ERR(entry); - nat_blk = folio_address(folio); + nat_blk = cache_address(entry); ne = nat_blk->entries[nid - start_nid]; node_info_from_raw_nat(ni, &ne); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); sanity_check: if (__is_valid_data_blkaddr(ni->blk_addr) && !f2fs_is_valid_blkaddr(sbi, ni->blk_addr, @@ -676,11 +678,11 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, } /* - * readahead MAX_RA_NODE number of node pages. + * readahead MAX_RA_NODE number of node caches. */ -static void f2fs_ra_node_pages(struct folio *parent, int start, int n) +static void f2fs_ra_node_caches(struct f2fs_cached_block *parent, int start, int n) { - struct f2fs_sb_info *sbi = F2FS_F_SB(parent); + struct f2fs_sb_info *sbi = parent->cache->sbi; struct blk_plug plug; int i, end; nid_t nid; @@ -689,10 +691,10 @@ static void f2fs_ra_node_pages(struct folio *parent, int start, int n) /* Then, try readahead for siblings of the desired node */ end = start + n; - end = min(end, (int)NIDS_PER_BLOCK); + end = min_t(int, end, NIDS_PER_BLOCK(sbi)); for (i = start; i < end; i++) { - nid = get_nid(parent, i, false); - f2fs_ra_node_page(sbi, nid); + nid = get_nid(sbi, parent, i, false); + f2fs_ra_node_cache(sbi, nid); } blk_finish_plug(&plug); @@ -700,9 +702,11 @@ static void f2fs_ra_node_pages(struct folio *parent, int start, int n) pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs) { + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); const long direct_index = ADDRS_PER_INODE(dn->inode); const long direct_blks = ADDRS_PER_BLOCK(dn->inode); - const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK; + const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * + NIDS_PER_BLOCK(sbi); unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode); int cur_level = dn->cur_level; int max_level = dn->max_level; @@ -712,7 +716,7 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs) return pgofs + 1; while (max_level-- > cur_level) - skipped_unit *= NIDS_PER_BLOCK; + skipped_unit *= NIDS_PER_BLOCK(sbi); switch (dn->max_level) { case 3: @@ -738,11 +742,13 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs) static int get_node_path(struct inode *inode, long block, int offset[4], unsigned int noffset[4]) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); const long direct_index = ADDRS_PER_INODE(inode); const long direct_blks = ADDRS_PER_BLOCK(inode); - const long dptrs_per_blk = NIDS_PER_BLOCK; - const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK; - const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK; + const long dptrs_per_blk = NIDS_PER_BLOCK(sbi); + const long indirect_blks = ADDRS_PER_BLOCK(inode) * + NIDS_PER_BLOCK(sbi); + const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK(sbi); int n = 0; int level = 0; @@ -754,7 +760,7 @@ static int get_node_path(struct inode *inode, long block, } block -= direct_index; if (block < direct_blks) { - offset[n++] = NODE_DIR1_BLOCK; + offset[n++] = NODE_DIR1_BLOCK(sbi); noffset[n] = 1; offset[n] = block; level = 1; @@ -762,7 +768,7 @@ static int get_node_path(struct inode *inode, long block, } block -= direct_blks; if (block < direct_blks) { - offset[n++] = NODE_DIR2_BLOCK; + offset[n++] = NODE_DIR2_BLOCK(sbi); noffset[n] = 2; offset[n] = block; level = 1; @@ -770,7 +776,7 @@ static int get_node_path(struct inode *inode, long block, } block -= direct_blks; if (block < indirect_blks) { - offset[n++] = NODE_IND1_BLOCK; + offset[n++] = NODE_IND1_BLOCK(sbi); noffset[n] = 3; offset[n++] = block / direct_blks; noffset[n] = 4 + offset[n - 1]; @@ -780,7 +786,7 @@ static int get_node_path(struct inode *inode, long block, } block -= indirect_blks; if (block < indirect_blks) { - offset[n++] = NODE_IND2_BLOCK; + offset[n++] = NODE_IND2_BLOCK(sbi); noffset[n] = 4 + dptrs_per_blk; offset[n++] = block / direct_blks; noffset[n] = 5 + dptrs_per_blk + offset[n - 1]; @@ -790,7 +796,7 @@ static int get_node_path(struct inode *inode, long block, } block -= indirect_blks; if (block < dindirect_blks) { - offset[n++] = NODE_DIND_BLOCK; + offset[n++] = NODE_DIND_BLOCK(sbi); noffset[n] = 5 + (dptrs_per_blk * 2); offset[n++] = block / indirect_blks; noffset[n] = 6 + (dptrs_per_blk * 2) + @@ -809,7 +815,8 @@ static int get_node_path(struct inode *inode, long block, return level; } -static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start); +static struct f2fs_cached_block *f2fs_get_node_cache_ra( + struct f2fs_cached_block *parent, int start); /* * Caller should call f2fs_put_dnode(dn). @@ -819,8 +826,8 @@ static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start); int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); - struct folio *nfolio[4]; - struct folio *parent = NULL; + struct f2fs_cached_block *nentry[4]; + struct f2fs_cached_block *parent = NULL; int offset[4]; unsigned int noffset[4]; nid_t nids[4]; @@ -833,26 +840,38 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) nids[0] = dn->inode->i_ino; - if (!dn->inode_folio) { - nfolio[0] = f2fs_get_inode_folio(sbi, nids[0]); - if (IS_ERR(nfolio[0])) - return PTR_ERR(nfolio[0]); + if (!dn->inode_entry) { + nentry[0] = f2fs_get_inode_cache(sbi, nids[0]); + if (IS_ERR(nentry[0])) { + err = PTR_ERR(nentry[0]); + /* + * cp_error drops dirty node caches, so the inode block + * of an inode that was never written reads as NEW_ADDR + * and returns -ENOENT. It is not a hole; the inode is + * gone. Callers treat -ENOENT as a hole and keep + * looking block by block (e.g. fiemap up to maxbytes). + */ + if (err == -ENOENT && (f2fs_cp_error(sbi) || + is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))) + err = -EIO; + return err; + } } else { - nfolio[0] = dn->inode_folio; + nentry[0] = dn->inode_entry; } /* if inline_data is set, should not report any block indices */ if (f2fs_has_inline_data(dn->inode) && index) { err = -ENOENT; - f2fs_folio_put(nfolio[0], true); + f2fs_put_cache(nentry[0], true); goto release_out; } - parent = nfolio[0]; + parent = nentry[0]; if (level != 0) - nids[1] = get_nid(parent, offset[0], true); - dn->inode_folio = nfolio[0]; - dn->inode_folio_locked = true; + nids[1] = get_nid(sbi, parent, offset[0], true); + dn->inode_entry = nentry[0]; + dn->inode_entry_locked = true; /* get indirect or direct nodes */ for (i = 1; i <= level; i++) { @@ -865,59 +884,59 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) "ino:%llu, nid:%u, level:%d, offset:%d", dn->inode->i_ino, nids[i], level, offset[level]); set_sbi_flag(sbi, SBI_NEED_FSCK); - goto release_pages; + goto release_caches; } if (!nids[i] && mode == ALLOC_NODE) { /* alloc new node */ if (!f2fs_alloc_nid(sbi, &(nids[i]))) { err = -ENOSPC; - goto release_pages; + goto release_caches; } dn->nid = nids[i]; - nfolio[i] = f2fs_new_node_folio(dn, noffset[i]); - if (IS_ERR(nfolio[i])) { + nentry[i] = f2fs_new_node_cache(dn, noffset[i]); + if (IS_ERR(nentry[i])) { f2fs_alloc_nid_failed(sbi, nids[i]); - err = PTR_ERR(nfolio[i]); - goto release_pages; + err = PTR_ERR(nentry[i]); + goto release_caches; } - set_nid(parent, offset[i - 1], nids[i], i == 1); + set_nid(sbi, parent, offset[i - 1], nids[i], i == 1); f2fs_alloc_nid_done(sbi, nids[i]); done = true; } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) { - nfolio[i] = f2fs_get_node_folio_ra(parent, offset[i - 1]); - if (IS_ERR(nfolio[i])) { - err = PTR_ERR(nfolio[i]); - goto release_pages; + nentry[i] = f2fs_get_node_cache_ra(parent, offset[i - 1]); + if (IS_ERR(nentry[i])) { + err = PTR_ERR(nentry[i]); + goto release_caches; } done = true; } if (i == 1) { - dn->inode_folio_locked = false; - folio_unlock(parent); + dn->inode_entry_locked = false; + f2fs_unlock_cache(parent); } else { - f2fs_folio_put(parent, true); + f2fs_put_cache(parent, true); } if (!done) { - nfolio[i] = f2fs_get_node_folio(sbi, nids[i], + nentry[i] = f2fs_get_node_cache(sbi, nids[i], NODE_TYPE_NON_INODE); - if (IS_ERR(nfolio[i])) { - err = PTR_ERR(nfolio[i]); - f2fs_folio_put(nfolio[0], false); + if (IS_ERR(nentry[i])) { + err = PTR_ERR(nentry[i]); + f2fs_put_cache(nentry[0], false); goto release_out; } } if (i < level) { - parent = nfolio[i]; - nids[i + 1] = get_nid(parent, offset[i], false); + parent = nentry[i]; + nids[i + 1] = get_nid(sbi, parent, offset[i], false); } } dn->nid = nids[level]; dn->ofs_in_node = offset[level]; - dn->node_folio = nfolio[level]; + dn->node_entry = nentry[level]; dn->data_blkaddr = f2fs_data_blkaddr(dn); if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) && @@ -938,9 +957,9 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) if (!c_len) goto out; - blkaddr = data_blkaddr(dn->inode, dn->node_folio, ofs_in_node); + blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node); if (blkaddr == COMPRESS_ADDR) - blkaddr = data_blkaddr(dn->inode, dn->node_folio, + blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node + 1); f2fs_update_read_extent_tree_range_compressed(dn->inode, @@ -949,13 +968,13 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) out: return 0; -release_pages: - f2fs_folio_put(parent, true); +release_caches: + f2fs_put_cache(parent, true); if (i > 1) - f2fs_folio_put(nfolio[0], false); + f2fs_put_cache(nentry[0], false); release_out: - dn->inode_folio = NULL; - dn->node_folio = NULL; + dn->inode_entry = NULL; + dn->node_entry = NULL; if (err == -ENOENT) { dn->cur_level = i; dn->max_level = level; @@ -996,16 +1015,15 @@ static int truncate_node(struct dnode_of_data *dn) f2fs_inode_synced(dn->inode); } - clear_node_folio_dirty(dn->node_folio); + f2fs_drop_cache_dirty(dn->node_entry); set_sbi_flag(sbi, SBI_IS_DIRTY); - index = dn->node_folio->index; - f2fs_folio_put(dn->node_folio, true); + index = dn->node_entry->index; + f2fs_put_cache(dn->node_entry, true); - invalidate_mapping_pages(NODE_MAPPING(sbi), - index, index); + f2fs_invalidate_node_cache(sbi, index); - dn->node_folio = NULL; + dn->node_entry = NULL; trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr); return 0; @@ -1014,35 +1032,35 @@ static int truncate_node(struct dnode_of_data *dn) static int truncate_dnode(struct dnode_of_data *dn) { struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); - struct folio *folio; + struct f2fs_cached_block *entry; int err; if (dn->nid == 0) return 1; /* get direct node */ - folio = f2fs_get_node_folio(sbi, dn->nid, NODE_TYPE_NON_INODE); - if (PTR_ERR(folio) == -ENOENT) + entry = f2fs_get_node_cache(sbi, dn->nid, NODE_TYPE_NON_INODE); + if (PTR_ERR(entry) == -ENOENT) return 1; - else if (IS_ERR(folio)) - return PTR_ERR(folio); + else if (IS_ERR(entry)) + return PTR_ERR(entry); - if (IS_INODE(folio) || ino_of_node(folio) != dn->inode->i_ino) { + if (IS_INODE(sbi, entry) || ino_of_node(sbi, entry) != dn->inode->i_ino) { f2fs_err(sbi, "incorrect node reference, ino: %llu, nid: %u, ino_of_node: %u", - dn->inode->i_ino, dn->nid, ino_of_node(folio)); + dn->inode->i_ino, dn->nid, ino_of_node(sbi, entry)); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return -EFSCORRUPTED; } /* Make dnode_of_data for parameter */ - dn->node_folio = folio; + dn->node_entry = entry; dn->ofs_in_node = 0; f2fs_truncate_data_blocks_range(dn, ADDRS_PER_BLOCK(dn->inode)); err = truncate_node(dn); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return err; } @@ -1052,8 +1070,9 @@ static int truncate_dnode(struct dnode_of_data *dn) static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, int ofs, int depth) { + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct dnode_of_data rdn = *dn; - struct folio *folio; + struct f2fs_cached_block *entry; struct f2fs_node *rn; nid_t child_nid; unsigned int child_nofs; @@ -1061,22 +1080,22 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, int i, ret; if (dn->nid == 0) - return NIDS_PER_BLOCK + 1; + return NIDS_PER_BLOCK(sbi) + 1; trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr); - folio = f2fs_get_node_folio(F2FS_I_SB(dn->inode), dn->nid, + entry = f2fs_get_node_cache(F2FS_I_SB(dn->inode), dn->nid, NODE_TYPE_NON_INODE); - if (IS_ERR(folio)) { - trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(folio)); - return PTR_ERR(folio); + if (IS_ERR(entry)) { + trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(entry)); + return PTR_ERR(entry); } - f2fs_ra_node_pages(folio, ofs, NIDS_PER_BLOCK); + f2fs_ra_node_caches(entry, ofs, NIDS_PER_BLOCK(sbi)); - rn = F2FS_NODE(folio); + rn = CACHED_NODE(entry); if (depth < 3) { - for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) { + for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++, freed++) { child_nid = le32_to_cpu(rn->in.nid[i]); if (child_nid == 0) continue; @@ -1084,21 +1103,21 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, ret = truncate_dnode(&rdn); if (ret < 0) goto out_err; - if (set_nid(folio, i, 0, false)) + if (set_nid(sbi, entry, i, 0, false)) dn->node_changed = true; } } else { - child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1; - for (i = ofs; i < NIDS_PER_BLOCK; i++) { + child_nofs = nofs + ofs * (NIDS_PER_BLOCK(sbi) + 1) + 1; + for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++) { child_nid = le32_to_cpu(rn->in.nid[i]); if (child_nid == 0) { - child_nofs += NIDS_PER_BLOCK + 1; + child_nofs += NIDS_PER_BLOCK(sbi) + 1; continue; } rdn.nid = child_nid; ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1); - if (ret == (NIDS_PER_BLOCK + 1)) { - if (set_nid(folio, i, 0, false)) + if (ret == (NIDS_PER_BLOCK(sbi) + 1)) { + if (set_nid(sbi, entry, i, 0, false)) dn->node_changed = true; child_nofs += ret; } else if (ret < 0 && ret != -ENOENT) { @@ -1110,19 +1129,19 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, if (!ofs) { /* remove current indirect node */ - dn->node_folio = folio; + dn->node_entry = entry; ret = truncate_node(dn); if (ret) goto out_err; freed++; } else { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } trace_f2fs_truncate_nodes_exit(dn->inode, freed); return freed; out_err: - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); trace_f2fs_truncate_nodes_exit(dn->inode, ret); return ret; } @@ -1130,60 +1149,62 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, static int truncate_partial_nodes(struct dnode_of_data *dn, int *offset, int depth) { - struct folio *folios[2]; + struct f2fs_cached_block *entries[2]; nid_t nid[3]; nid_t child_nid; int err = 0; int i; int idx = depth - 2; - nid[0] = get_nid(dn->inode_folio, offset[0], true); + nid[0] = get_nid(F2FS_I_SB(dn->inode), dn->inode_entry, offset[0], true); if (!nid[0]) return 0; /* get indirect nodes in the path */ for (i = 0; i < idx + 1; i++) { /* reference count'll be increased */ - folios[i] = f2fs_get_node_folio(F2FS_I_SB(dn->inode), nid[i], + entries[i] = f2fs_get_node_cache(F2FS_I_SB(dn->inode), nid[i], NODE_TYPE_NON_INODE); - if (IS_ERR(folios[i])) { - err = PTR_ERR(folios[i]); + if (IS_ERR(entries[i])) { + err = PTR_ERR(entries[i]); idx = i - 1; goto fail; } - nid[i + 1] = get_nid(folios[i], offset[i + 1], false); + nid[i + 1] = get_nid(F2FS_I_SB(dn->inode), entries[i], offset[i + 1], false); } - f2fs_ra_node_pages(folios[idx], offset[idx + 1], NIDS_PER_BLOCK); + f2fs_ra_node_caches(entries[idx], offset[idx + 1], + NIDS_PER_BLOCK(F2FS_I_SB(dn->inode))); /* free direct nodes linked to a partial indirect node */ - for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) { - child_nid = get_nid(folios[idx], i, false); + for (i = offset[idx + 1]; + i < NIDS_PER_BLOCK(F2FS_I_SB(dn->inode)); i++) { + child_nid = get_nid(F2FS_I_SB(dn->inode), entries[idx], i, false); if (!child_nid) continue; dn->nid = child_nid; err = truncate_dnode(dn); if (err < 0) goto fail; - if (set_nid(folios[idx], i, 0, false)) + if (set_nid(F2FS_I_SB(dn->inode), entries[idx], i, 0, false)) dn->node_changed = true; } if (offset[idx + 1] == 0) { - dn->node_folio = folios[idx]; + dn->node_entry = entries[idx]; dn->nid = nid[idx]; err = truncate_node(dn); if (err) goto fail; } else { - f2fs_folio_put(folios[idx], true); + f2fs_put_cache(entries[idx], true); } offset[idx]++; offset[idx + 1] = 0; idx--; fail: for (i = idx; i >= 0; i--) - f2fs_folio_put(folios[i], true); + f2fs_put_cache(entries[i], true); trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err); @@ -1191,7 +1212,9 @@ static int truncate_partial_nodes(struct dnode_of_data *dn, } /* - * All the block addresses of data and nodes should be nullified. + * All the node blocks actually belong to the inode will be released. + * If the level is 0, we will simply truncate the dnode, + * or else we should do dynamic truncate for the node pointers with the depth. */ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) { @@ -1200,7 +1223,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) int level, offset[4], noffset[4]; unsigned int nofs = 0; struct dnode_of_data dn; - struct folio *folio; + struct f2fs_cached_block *entry; trace_f2fs_truncate_inode_blocks_enter(inode, from); @@ -1217,14 +1240,14 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) return level; } - folio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(folio)) { - trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(folio)); - return PTR_ERR(folio); + entry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(entry)) { + trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(entry)); + return PTR_ERR(entry); } - set_new_dnode(&dn, inode, folio, NULL, 0); - folio_unlock(folio); + set_new_dnode(&dn, inode, entry, NULL, 0); + f2fs_unlock_cache(entry); switch (level) { case 0: @@ -1238,10 +1261,10 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) err = truncate_partial_nodes(&dn, offset, level); if (err < 0 && err != -ENOENT) goto fail; - nofs += 1 + NIDS_PER_BLOCK; + nofs += 1 + NIDS_PER_BLOCK(sbi); break; case 3: - nofs = 5 + 2 * NIDS_PER_BLOCK; + nofs = 5 + 2 * NIDS_PER_BLOCK(sbi); if (!offset[level - 1]) goto skip_partial; err = truncate_partial_nodes(&dn, offset, level); @@ -1254,28 +1277,21 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) skip_partial: while (cont) { - dn.nid = get_nid(folio, offset[0], true); - switch (offset[0]) { - case NODE_DIR1_BLOCK: - case NODE_DIR2_BLOCK: + dn.nid = get_nid(sbi, entry, offset[0], true); + if (offset[0] == NODE_DIR1_BLOCK(sbi) || + offset[0] == NODE_DIR2_BLOCK(sbi)) { err = truncate_dnode(&dn); - break; - - case NODE_IND1_BLOCK: - case NODE_IND2_BLOCK: + } else if (offset[0] == NODE_IND1_BLOCK(sbi) || + offset[0] == NODE_IND2_BLOCK(sbi)) { err = truncate_nodes(&dn, nofs, offset[1], 2); - break; - - case NODE_DIND_BLOCK: + } else if (offset[0] == NODE_DIND_BLOCK(sbi)) { err = truncate_nodes(&dn, nofs, offset[1], 3); cont = 0; - break; - - default: + } else { BUG(); } if (err == -ENOENT) { - set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK); + set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); fserror_report_file_metadata(dn.inode, -EFSCORRUPTED, GFP_NOFS); @@ -1288,42 +1304,42 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) } if (err < 0) goto fail; - if (offset[1] == 0 && get_nid(folio, offset[0], true)) { - folio_lock(folio); - BUG_ON(!is_node_folio(folio)); - set_nid(folio, offset[0], 0, true); - folio_unlock(folio); + if (offset[1] == 0 && get_nid(sbi, entry, offset[0], true)) { + f2fs_lock_cache(entry); + f2fs_bug_on(sbi, !f2fs_is_node_cache(entry)); + set_nid(sbi, entry, offset[0], 0, true); + f2fs_unlock_cache(entry); } offset[1] = 0; offset[0]++; nofs += err; } fail: - f2fs_folio_put(folio, false); + f2fs_put_cache(entry, false); trace_f2fs_truncate_inode_blocks_exit(inode, err); return err > 0 ? 0 : err; } -/* caller must lock inode page */ +/* caller must lock inode cached block */ int f2fs_truncate_xattr_node(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); nid_t nid = F2FS_I(inode)->i_xattr_nid; struct dnode_of_data dn; - struct folio *nfolio; + struct f2fs_cached_block *nentry; int err; if (!nid) return 0; - nfolio = f2fs_get_xnode_folio(sbi, nid); - if (IS_ERR(nfolio)) - return PTR_ERR(nfolio); + nentry = f2fs_get_xnode_cache(sbi, nid); + if (IS_ERR(nentry)) + return PTR_ERR(nentry); - set_new_dnode(&dn, inode, NULL, nfolio, nid); + set_new_dnode(&dn, inode, NULL, nentry, nid); err = truncate_node(&dn); if (err) { - f2fs_folio_put(nfolio, true); + f2fs_put_cache(nentry, true); return err; } @@ -1336,7 +1352,7 @@ int f2fs_truncate_xattr_node(struct inode *inode) * Caller should grab and release a rwsem by calling f2fs_lock_op() and * f2fs_unlock_op(). */ -int f2fs_remove_inode_page(struct inode *inode) +int f2fs_remove_inode_cache(struct inode *inode) { struct dnode_of_data dn; int err; @@ -1366,12 +1382,12 @@ int f2fs_remove_inode_page(struct inode *inode) if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) { f2fs_warn(F2FS_I_SB(inode), - "f2fs_remove_inode_page: inconsistent i_blocks, ino:%llu, iblocks:%llu", + "f2fs_remove_inode_cache: inconsistent i_blocks, ino:%llu, iblocks:%llu", inode->i_ino, (unsigned long long)inode->i_blocks); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); } - /* will put inode & node pages */ + /* will put inode & node cached blocks */ err = truncate_node(&dn); if (err) { f2fs_put_dnode(&dn); @@ -1380,30 +1396,30 @@ int f2fs_remove_inode_page(struct inode *inode) return 0; } -struct folio *f2fs_new_inode_folio(struct inode *inode) +struct f2fs_cached_block *f2fs_new_inode_cache(struct inode *inode) { struct dnode_of_data dn; - /* allocate inode page for new inode */ + /* allocate inode block for new inode */ set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); - /* caller should f2fs_folio_put(folio, true); */ - return f2fs_new_node_folio(&dn, 0); + /* caller should f2fs_put_cache(entry, true); */ + return f2fs_new_node_cache(&dn, 0); } -struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs) +struct f2fs_cached_block *f2fs_new_node_cache(struct dnode_of_data *dn, unsigned int ofs) { struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct node_info new_ni; - struct folio *folio; + struct f2fs_cached_block *entry; int err; if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) return ERR_PTR(-EPERM); - folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), dn->nid, false); - if (IS_ERR(folio)) - return folio; + entry = f2fs_grab_node_cache(sbi, dn->nid); + if (IS_ERR(entry)) + return entry; if (unlikely((err = inc_valid_node_count(sbi, dn->inode, !ofs)))) goto fail; @@ -1419,7 +1435,7 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs) dec_valid_node_count(sbi, dn->inode, !ofs); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn_ratelimited(sbi, - "f2fs_new_node_folio: inconsistent nat entry, " + "f2fs_new_node_cache: inconsistent nat entry, " "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", new_ni.ino, new_ni.nid, new_ni.blk_addr, new_ni.version, new_ni.flag); @@ -1434,12 +1450,11 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs) new_ni.version = 0; set_node_addr(sbi, &new_ni, NEW_ADDR, false); - f2fs_folio_wait_writeback(folio, NODE, true, true); - fill_node_footer(folio, dn->nid, dn->inode->i_ino, ofs, true); - set_cold_node(folio, S_ISDIR(dn->inode->i_mode)); - if (!folio_test_uptodate(folio)) - folio_mark_uptodate(folio); - if (folio_mark_dirty(folio)) + f2fs_cache_wait_writeback(entry); + fill_node_footer(sbi, entry, dn->nid, dn->inode->i_ino, ofs, true); + set_cold_node(sbi, entry, S_ISDIR(dn->inode->i_mode)); + f2fs_cache_set_uptodate(entry); + if (f2fs_mark_cache_dirty(entry)) dn->node_changed = true; if (f2fs_has_xattr_block(ofs)) @@ -1447,66 +1462,67 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs) if (ofs == 0) inc_valid_inode_count(sbi); - return folio; + return entry; fail: - clear_node_folio_dirty(folio); - f2fs_folio_put(folio, true); + f2fs_drop_cache_dirty(entry); + f2fs_put_cache(entry, true); return ERR_PTR(err); } /* * Caller should do after getting the following values. - * 0: f2fs_folio_put(folio, false) - * LOCKED_PAGE or error: f2fs_folio_put(folio, true) + * 0: f2fs_put_cache(cache, false) + * LOCKED_CACHE or error: f2fs_put_cache(entry, true) */ -static int read_node_folio(struct folio *folio, blk_opf_t op_flags) +static int read_node_cache(struct f2fs_cached_block *entry, blk_opf_t op_flags) { - struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct f2fs_sb_info *sbi = entry->cache->sbi; struct node_info ni; struct f2fs_io_info fio = { .sbi = sbi, .type = NODE, .op = REQ_OP_READ, .op_flags = op_flags, - .folio = folio, .encrypted_page = NULL, + .cache_entry = entry, + .is_cache = 1, }; int err; - if (folio_test_uptodate(folio)) { - if (!f2fs_inode_chksum_verify(sbi, folio)) { - folio_clear_uptodate(folio); + if (f2fs_cache_test_uptodate(entry)) { + if (!f2fs_inode_chksum_verify(sbi, entry)) { + f2fs_cache_clear_uptodate(entry); return -EFSBADCRC; } - return LOCKED_PAGE; + return LOCKED_CACHE; } - err = f2fs_get_node_info(sbi, folio->index, &ni, false); + err = f2fs_get_node_info(sbi, entry->index, &ni, false); if (err) return err; - /* NEW_ADDR can be seen, after cp_error drops some dirty node pages */ + /* NEW_ADDR can be seen, after cp_error drops some dirty node caches */ if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) { - folio_clear_uptodate(folio); + f2fs_cache_clear_uptodate(entry); return -ENOENT; } fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr; - err = f2fs_submit_page_bio(&fio); - + err = f2fs_submit_cache_read(&fio); if (!err) - f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, + F2FS_BLKSIZE(sbi)); return err; } /* - * Readahead a node page + * Readahead a node cache */ -void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) +void f2fs_ra_node_cache(struct f2fs_sb_info *sbi, nid_t nid) { - struct folio *afolio; + struct f2fs_cached_block *entry; int err; if (!nid) @@ -1514,29 +1530,29 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) if (f2fs_check_nid_range(sbi, nid)) return; - afolio = xa_load(&NODE_MAPPING(sbi)->i_pages, nid); - if (afolio) + entry = xa_load(&NODE_CACHE(sbi)->root, nid); + if (entry) return; - afolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false); - if (IS_ERR(afolio)) + entry = f2fs_grab_node_cache(sbi, nid); + if (IS_ERR(entry)) return; - err = read_node_folio(afolio, REQ_RAHEAD); - f2fs_folio_put(afolio, err ? true : false); + err = read_node_cache(entry, REQ_RAHEAD); + f2fs_put_cache(entry, err ? true : false); } int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, - struct folio *folio, pgoff_t nid, + struct f2fs_cached_block *entry, pgoff_t nid, enum node_type ntype, bool in_irq) { bool is_inode, is_xnode; - if (unlikely(nid != nid_of_node(folio))) + if (unlikely(nid != nid_of_node(sbi, entry))) goto out_err; - is_inode = IS_INODE(folio); - is_xnode = f2fs_has_xattr_block(ofs_of_node(folio)); + is_inode = IS_INODE(sbi, entry); + is_xnode = f2fs_has_xattr_block(ofs_of_node(sbi, entry)); switch (ntype) { case NODE_TYPE_REGULAR: @@ -1569,20 +1585,20 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", - ntype, nid, nid_of_node(folio), ino_of_node(folio), - ofs_of_node(folio), cpver_of_node(folio), - next_blkaddr_of_node(folio)); + ntype, nid, nid_of_node(sbi, entry), ino_of_node(sbi, entry), + ofs_of_node(sbi, entry), cpver_of_node(sbi, entry), + next_blkaddr_of_node(sbi, entry)); f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); - fserror_report_file_metadata(folio->mapping->host, - -EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS); + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, + in_irq ? GFP_NOWAIT : GFP_NOFS); return -EFSCORRUPTED; } -static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, - struct folio *parent, int start, enum node_type ntype) +static struct f2fs_cached_block *__get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid, + struct f2fs_cached_block *parent, int start, enum node_type ntype) { - struct folio *folio; + struct f2fs_cached_block *entry; int err; if (!nid) @@ -1590,71 +1606,71 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, if (f2fs_check_nid_range(sbi, nid)) return ERR_PTR(-EINVAL); repeat: - folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false); - if (IS_ERR(folio)) - return folio; + entry = f2fs_grab_node_cache(sbi, nid); + if (IS_ERR(entry)) + return entry; - err = read_node_folio(folio, 0); + err = read_node_cache(entry, 0); if (err < 0) goto out_put_err; - if (err == LOCKED_PAGE) - goto page_hit; + if (err == LOCKED_CACHE) + goto entry_hit; if (parent) - f2fs_ra_node_pages(parent, start + 1, MAX_RA_NODE); + f2fs_ra_node_caches(parent, start + 1, MAX_RA_NODE); - folio_lock(folio); + f2fs_lock_cache(entry); - if (unlikely(!is_node_folio(folio))) { - f2fs_folio_put(folio, true); + if (unlikely(!f2fs_is_node_cache(entry))) { + f2fs_put_cache(entry, true); goto repeat; } - if (unlikely(!folio_test_uptodate(folio))) { + if (unlikely(!f2fs_cache_test_uptodate(entry))) { err = -EIO; goto out_put_err; } - if (!f2fs_inode_chksum_verify(sbi, folio)) { + if (!f2fs_inode_chksum_verify(sbi, entry)) { err = -EFSBADCRC; goto out_err; } -page_hit: - err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); +entry_hit: + err = f2fs_sanity_check_node_footer(sbi, entry, nid, ntype, false); if (!err) - return folio; + return entry; out_err: - clear_node_folio_dirty(folio); + f2fs_drop_cache_dirty(entry); out_put_err: /* ENOENT comes from read_node_folio which is not an error. */ if (err != -ENOENT) - f2fs_handle_page_eio(sbi, folio, NODE); - f2fs_folio_put(folio, true); + f2fs_handle_page_eio(sbi, entry->index, NODE); + f2fs_put_cache(entry, true); return ERR_PTR(err); } -struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, +struct f2fs_cached_block *f2fs_get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type) { - return __get_node_folio(sbi, nid, NULL, 0, node_type); + return __get_node_cache(sbi, nid, NULL, 0, node_type); } -struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino) +struct f2fs_cached_block *f2fs_get_inode_cache(struct f2fs_sb_info *sbi, pgoff_t ino) { - return __get_node_folio(sbi, ino, NULL, 0, NODE_TYPE_INODE); + return __get_node_cache(sbi, ino, NULL, 0, NODE_TYPE_INODE); } -struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid) +struct f2fs_cached_block *f2fs_get_xnode_cache(struct f2fs_sb_info *sbi, pgoff_t xnid) { - return __get_node_folio(sbi, xnid, NULL, 0, NODE_TYPE_XATTR); + return __get_node_cache(sbi, xnid, NULL, 0, NODE_TYPE_XATTR); } -static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start) +static struct f2fs_cached_block *f2fs_get_node_cache_ra(struct f2fs_cached_block *parent, int start) { - struct f2fs_sb_info *sbi = F2FS_F_SB(parent); - nid_t nid = get_nid(parent, start, false); + struct f2fs_sb_info *sbi = parent->cache->sbi; + nid_t nid = get_nid(sbi, parent, start, false); - return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE); + return __get_node_cache(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE); } static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) @@ -1693,110 +1709,106 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) iput(inode); } -static struct folio *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) +static struct f2fs_cached_block *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) { - pgoff_t index; - struct folio_batch fbatch; - struct folio *last_folio = NULL; - int nr_folios; + pgoff_t index = 0; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; + struct f2fs_cached_block *last_entry = NULL; + unsigned int nr; - folio_batch_init(&fbatch); - index = 0; - - while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, - (pgoff_t)-1, PAGECACHE_TAG_DIRTY, - &fbatch))) { + while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), + entries, &index, F2FS_ONSTACK_CACHES, + F2FS_CACHE_TAG_DIRTY))) { int i; - for (i = 0; i < nr_folios; i++) { - struct folio *folio = fbatch.folios[i]; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; if (unlikely(f2fs_cp_error(sbi))) { - f2fs_folio_put(last_folio, false); - folio_batch_release(&fbatch); + f2fs_put_cache(last_entry, false); + f2fs_cache_gang_release(entries, nr); return ERR_PTR(-EIO); } - if (!IS_DNODE(folio) || !is_cold_node(folio)) + if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry)) continue; - if (ino_of_node(folio) != ino) + if (ino_of_node(sbi, entry) != ino) continue; - folio_lock(folio); + f2fs_lock_cache(entry); - if (unlikely(!is_node_folio(folio))) { + if (unlikely(!f2fs_is_node_cache(entry))) { continue_unlock: - folio_unlock(folio); + f2fs_unlock_cache(entry); continue; } - if (ino_of_node(folio) != ino) + if (ino_of_node(sbi, entry) != ino) goto continue_unlock; - if (!folio_test_dirty(folio)) { + if (!f2fs_cache_test_dirty(entry)) { /* someone wrote it for us */ goto continue_unlock; } - if (last_folio) - f2fs_folio_put(last_folio, false); + if (last_entry) + f2fs_put_cache(last_entry, false); - folio_get(folio); - last_folio = folio; - folio_unlock(folio); + f2fs_cache_get(entry); + last_entry = entry; + f2fs_unlock_cache(entry); } - folio_batch_release(&fbatch); + f2fs_cache_gang_release(entries, nr); cond_resched(); } - return last_folio; + return last_entry; } -static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, - bool *submitted, struct writeback_control *wbc, - bool do_balance, enum iostat_type io_type, - unsigned int *seq_id) +static bool __write_node_cache(struct f2fs_cached_block *entry, + bool atomic, bool do_fsync, bool *submitted, + bool sync, bool do_balance, + enum iostat_type io_type, unsigned int *seq_id) { - struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct f2fs_sb_info *sbi = entry->cache->sbi; nid_t nid; struct node_info ni; struct f2fs_io_info fio = { .sbi = sbi, - .ino = ino_of_node(folio), + .ino = ino_of_node(sbi, entry), .type = NODE, .op = REQ_OP_WRITE, - .op_flags = wbc_to_write_flags(wbc), - .folio = folio, + .op_flags = sync ? REQ_SYNC : REQ_BACKGROUND, + .cache_entry = entry, .encrypted_page = NULL, .submitted = 0, .io_type = io_type, - .io_wbc = wbc, + .is_cache = true, }; struct f2fs_lock_context lc; unsigned int seq; - trace_f2fs_writepage(folio, NODE); - if (unlikely(f2fs_cp_error(sbi))) { - /* keep node pages in remount-ro mode */ + /* keep node caches in remount-ro mode */ if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) goto redirty_out; - folio_clear_uptodate(folio); - dec_page_count(sbi, F2FS_DIRTY_NODES); - folio_unlock(folio); + f2fs_cache_clear_uptodate(entry); + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, + F2FS_CACHE_TAG_NONE); + dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES); + f2fs_unlock_cache(entry); return true; } if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) goto redirty_out; - if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && - wbc->sync_mode == WB_SYNC_NONE && - IS_DNODE(folio) && is_cold_node(folio)) + if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && !sync && + IS_DNODE(sbi, entry) && is_cold_node(sbi, entry)) goto redirty_out; - /* get old block addr of this node page */ - nid = nid_of_node(folio); + /* get old block addr of this node cache */ + nid = nid_of_node(sbi, entry); - if (f2fs_sanity_check_node_footer(sbi, folio, folio->index, + if (f2fs_sanity_check_node_footer(sbi, entry, entry->index, NODE_TYPE_REGULAR, false)) { fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); @@ -1808,12 +1820,14 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, f2fs_down_read_trace(&sbi->node_write, &lc); - /* This page is already truncated */ + /* This cache is already truncated */ if (unlikely(ni.blk_addr == NULL_ADDR)) { - folio_clear_uptodate(folio); - dec_page_count(sbi, F2FS_DIRTY_NODES); + f2fs_cache_clear_uptodate(entry); + f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, + F2FS_CACHE_TAG_NONE); + dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES); f2fs_up_read_trace(&sbi->node_write, &lc); - folio_unlock(folio); + f2fs_unlock_cache(entry); return true; } @@ -1827,28 +1841,28 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, if (atomic && !test_opt(sbi, NOBARRIER)) fio.op_flags |= REQ_PREFLUSH | REQ_FUA; - set_dentry_mark(folio, false); - set_fsync_mark(folio, do_fsync); - if (IS_INODE(folio) && (atomic || is_fsync_dnode(folio))) - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + set_dentry_mark(sbi, entry, false); + set_fsync_mark(sbi, entry, do_fsync); + if (IS_INODE(sbi, entry) && (atomic || is_fsync_dnode(sbi, entry))) + set_dentry_mark(sbi, entry, + f2fs_need_dentry_mark(sbi, ino_of_node(sbi, entry))); - /* should add to global list before clearing PAGECACHE status */ - if (f2fs_in_warm_node_list(folio)) { - seq = f2fs_add_fsync_node_entry(sbi, folio); + /* should add to global list before clearing CACHE status */ + if (f2fs_in_warm_node_list(sbi, entry)) { + seq = f2fs_add_fsync_node_entry(sbi, entry); if (seq_id) *seq_id = seq; } - folio_start_writeback(folio); + f2fs_start_cache_writeback(entry); fio.old_blkaddr = ni.blk_addr; - f2fs_do_write_node_page(nid, &fio); - set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(folio)); - dec_page_count(sbi, F2FS_DIRTY_NODES); + f2fs_do_write_node_cache(nid, &fio); + set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(sbi, entry)); + dec_cache_count(sbi, F2FS_DIRTY_NODES); f2fs_up_read_trace(&sbi->node_write, &lc); - folio_unlock(folio); + f2fs_unlock_cache(entry); if (unlikely(f2fs_cp_error(sbi))) { f2fs_submit_merged_write(sbi, NODE); @@ -1862,176 +1876,171 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, return true; redirty_out: - folio_redirty_for_writepage(wbc, folio); - folio_unlock(folio); + f2fs_cache_set_dirty(entry); + f2fs_unlock_cache(entry); return false; } -int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, +int f2fs_write_node_cache(struct f2fs_cached_block *node_entry, int sync_mode, bool mark_dirty, enum iostat_type io_type) { int err = 0; - struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, - .nr_to_write = 1, - }; if (!sync_mode) { - /* set page dirty and write it */ - if (!folio_test_writeback(node_folio)) - folio_mark_dirty(node_folio); - goto out_folio; + /* set cache dirty and write it */ + if (!f2fs_cache_test_writeback(node_entry)) + f2fs_mark_cache_dirty(node_entry); + goto out_entry; } - f2fs_folio_wait_writeback(node_folio, NODE, true, true); + f2fs_cache_wait_writeback(node_entry); if (mark_dirty) - folio_mark_dirty(node_folio); - else if (!folio_test_dirty(node_folio)) - goto out_folio; + f2fs_mark_cache_dirty(node_entry); + else if (!f2fs_cache_test_dirty(node_entry)) + goto out_entry; - if (!folio_clear_dirty_for_io(node_folio)) { + if (!f2fs_cache_test_and_clear_dirty(node_entry)) { err = -EAGAIN; - goto out_folio; + goto out_entry; } - if (!__write_node_folio(node_folio, false, false, NULL, - &wbc, false, io_type, NULL)) + if (!__write_node_cache(node_entry, false, false, NULL, + true, false, io_type, NULL)) err = -EAGAIN; - goto release_folio; -out_folio: - folio_unlock(node_folio); -release_folio: - f2fs_folio_put(node_folio, false); + goto release_entry; +out_entry: + f2fs_unlock_cache(node_entry); +release_entry: + f2fs_put_cache(node_entry, false); return err; } -int f2fs_move_node_folio(struct folio *node_folio, int gc_type) +int f2fs_move_node_cache(struct f2fs_cached_block *entry, int gc_type) { - return f2fs_write_single_node_folio(node_folio, gc_type == FG_GC, + return f2fs_write_node_cache(entry, gc_type == FG_GC, true, FS_GC_NODE_IO); } -int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - struct writeback_control *wbc, bool atomic, - unsigned int *seq_id) +int f2fs_fsync_node_caches(struct f2fs_sb_info *sbi, struct inode *inode, + bool atomic, unsigned int *seq_id) { pgoff_t index; - struct folio_batch fbatch; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; int ret = 0; - struct folio *last_folio = NULL; + struct f2fs_cached_block *last_entry = NULL; bool marked = false; nid_t ino = inode->i_ino; - int nr_folios; + int nr; int nwritten = 0; if (atomic) { - last_folio = last_fsync_dnode(sbi, ino); - if (IS_ERR_OR_NULL(last_folio)) - return PTR_ERR_OR_ZERO(last_folio); + last_entry = last_fsync_dnode(sbi, ino); + if (IS_ERR_OR_NULL(last_entry)) + return PTR_ERR_OR_ZERO(last_entry); } retry: - folio_batch_init(&fbatch); index = 0; - while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, - (pgoff_t)-1, PAGECACHE_TAG_DIRTY, - &fbatch))) { + while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), + entries, &index, F2FS_ONSTACK_CACHES, + F2FS_CACHE_TAG_DIRTY))) { int i; - for (i = 0; i < nr_folios; i++) { - struct folio *folio = fbatch.folios[i]; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; bool submitted = false; bool do_fsync = false; if (unlikely(f2fs_cp_error(sbi))) { - f2fs_folio_put(last_folio, false); - folio_batch_release(&fbatch); + f2fs_put_cache(last_entry, false); + f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; } - if (!IS_DNODE(folio) || !is_cold_node(folio)) + if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry)) continue; - if (ino_of_node(folio) != ino) + if (ino_of_node(sbi, entry) != ino) continue; - folio_lock(folio); + f2fs_lock_cache(entry); - if (unlikely(!is_node_folio(folio))) { + if (unlikely(!f2fs_is_node_cache(entry))) { continue_unlock: - folio_unlock(folio); + f2fs_unlock_cache(entry); continue; } - if (ino_of_node(folio) != ino) + if (ino_of_node(sbi, entry) != ino) goto continue_unlock; - if (!folio_test_dirty(folio) && folio != last_folio) { + if (!f2fs_cache_test_dirty(entry) && entry != last_entry) { /* someone wrote it for us */ goto continue_unlock; } - f2fs_folio_wait_writeback(folio, NODE, true, true); + f2fs_cache_wait_writeback(entry); - if (!atomic || folio == last_folio) { + if (!atomic || entry == last_entry) { do_fsync = true; percpu_counter_inc(&sbi->rf_node_block_count); - if (IS_INODE(folio)) { + if (IS_INODE(sbi, entry)) { if (is_inode_flag_set(inode, FI_DIRTY_INODE)) - f2fs_update_inode(inode, folio); + f2fs_update_inode(inode, entry); } /* may be written by other thread */ - if (!folio_test_dirty(folio)) - folio_mark_dirty(folio); + if (!f2fs_cache_test_dirty(entry)) + f2fs_mark_cache_dirty(entry); } - if (!folio_clear_dirty_for_io(folio)) + if (!f2fs_cache_test_and_clear_dirty(entry)) goto continue_unlock; - if (!__write_node_folio(folio, atomic && - folio == last_folio, + if (!__write_node_cache(entry, atomic && + entry == last_entry, do_fsync, &submitted, - wbc, true, FS_NODE_IO, + true, true, FS_NODE_IO, seq_id)) { - f2fs_folio_put(last_folio, false); - folio_batch_release(&fbatch); + f2fs_put_cache(last_entry, false); + f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; } if (submitted) nwritten++; - if (folio == last_folio) { - f2fs_folio_put(folio, false); - folio_batch_release(&fbatch); + if (entry == last_entry) { + f2fs_put_cache(entry, false); + f2fs_cache_gang_release(entries, nr); marked = true; goto out; } } - folio_batch_release(&fbatch); + f2fs_cache_gang_release(entries, nr); cond_resched(); } if (atomic && !marked) { f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", - ino, last_folio->index); - folio_lock(last_folio); - if (unlikely(!is_node_folio(last_folio))) { - f2fs_folio_put(last_folio, true); + ino, last_entry->index); + f2fs_lock_cache(last_entry); + if (unlikely(!f2fs_is_node_cache(last_entry))) { + f2fs_put_cache(last_entry, true); ret = -EAGAIN; goto out; } - f2fs_folio_wait_writeback(last_folio, NODE, true, true); - folio_mark_dirty(last_folio); - folio_unlock(last_folio); + f2fs_cache_wait_writeback(last_entry); + f2fs_mark_cache_dirty(last_entry); + f2fs_unlock_cache(last_entry); goto retry; } out: if (nwritten) - f2fs_submit_merged_write_cond(sbi, NULL, NULL, ino, NODE); + f2fs_submit_merged_write_cache(sbi, NULL, ino, NODE); return ret; } + static int f2fs_match_ino(struct inode *inode, u64 ino, void *data) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); @@ -2056,18 +2065,18 @@ static int f2fs_match_ino(struct inode *inode, u64 ino, void *data) return 1; } -static bool flush_dirty_inode(struct folio *folio) +static bool flush_dirty_cache_inode(struct f2fs_cached_block *entry) { - struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct f2fs_sb_info *sbi = entry->cache->sbi; struct inode *inode; - nid_t ino = ino_of_node(folio); + nid_t ino = ino_of_node(sbi, entry); inode = find_inode_nowait(sbi->sb, ino, f2fs_match_ino, NULL); if (!inode) return false; - f2fs_update_inode(inode, folio); - folio_unlock(folio); + f2fs_update_inode(inode, entry); + f2fs_unlock_cache(entry); iput(inode); return true; @@ -2076,72 +2085,75 @@ static bool flush_dirty_inode(struct folio *folio) void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) { pgoff_t index = 0; - struct folio_batch fbatch; - int nr_folios; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; + unsigned int nr; - folio_batch_init(&fbatch); - - while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, - (pgoff_t)-1, PAGECACHE_TAG_DIRTY, - &fbatch))) { + while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), + entries, &index, F2FS_ONSTACK_CACHES, + F2FS_CACHE_TAG_DIRTY))) { int i; - for (i = 0; i < nr_folios; i++) { - struct folio *folio = fbatch.folios[i]; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; - if (!IS_INODE(folio)) - continue; + f2fs_lock_cache(entry); - folio_lock(folio); - - if (unlikely(!is_node_folio(folio))) + if (unlikely(!f2fs_is_node_cache(entry))) goto unlock; - if (!folio_test_dirty(folio)) + if (!IS_INODE(sbi, entry)) + goto unlock; + if (!f2fs_cache_test_dirty(entry)) goto unlock; /* flush inline_data, if it's async context. */ - if (folio_test_f2fs_inline(folio)) { - folio_clear_f2fs_inline(folio); - folio_unlock(folio); - flush_inline_data(sbi, ino_of_node(folio)); + if (f2fs_cache_test_inline(entry)) { + nid_t ino = ino_of_node(sbi, entry); + + f2fs_cache_clear_inline(entry); + f2fs_unlock_cache(entry); + flush_inline_data(sbi, ino); continue; } unlock: - folio_unlock(folio); + f2fs_unlock_cache(entry); } - folio_batch_release(&fbatch); + f2fs_cache_gang_release(entries, nr); cond_resched(); } } -int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - struct writeback_control *wbc, - bool do_balance, enum iostat_type io_type) +int f2fs_writeback_node_caches(struct f2fs_sb_info *sbi, long nr_to_write, + bool sync, bool do_balance, enum iostat_type io_type, + long *nr_written) { pgoff_t index; - struct folio_batch fbatch; + struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; int step = 0; int nwritten = 0; int ret = 0; - int nr_folios, done = 0; + int nr, done = 0; - folio_batch_init(&fbatch); + trace_f2fs_write_caches(sbi, nr_to_write, 0, NODE); next_step: index = 0; - while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), - &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY, - &fbatch))) { + while (!done && (nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), + entries, &index, F2FS_ONSTACK_CACHES, + F2FS_CACHE_TAG_DIRTY))) { int i; - for (i = 0; i < nr_folios; i++) { - struct folio *folio = fbatch.folios[i]; + for (i = 0; i < nr; i++) { + struct f2fs_cached_block *entry = entries[i]; bool submitted = false; + if (!sync && unlikely(freezing(current))) { + done = 1; + break; + } + /* give a priority to WB_SYNC threads */ - if (atomic_read(&sbi->wb_sync_req[NODE]) && - wbc->sync_mode == WB_SYNC_NONE) { + if (atomic_read(&sbi->wb_sync_req[NODE]) && !sync) { done = 1; break; } @@ -2152,27 +2164,27 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, * 1. dentry dnodes * 2. file dnodes */ - if (step == 0 && IS_DNODE(folio)) + if (step == 0 && IS_DNODE(sbi, entry)) continue; - if (step == 1 && (!IS_DNODE(folio) || - is_cold_node(folio))) + if (step == 1 && (!IS_DNODE(sbi, entry) || + is_cold_node(sbi, entry))) continue; - if (step == 2 && (!IS_DNODE(folio) || - !is_cold_node(folio))) + if (step == 2 && (!IS_DNODE(sbi, entry) || + !is_cold_node(sbi, entry))) continue; lock_node: - if (wbc->sync_mode == WB_SYNC_ALL) - folio_lock(folio); - else if (!folio_trylock(folio)) + if (sync) + f2fs_lock_cache(entry); + else if (!f2fs_trylock_cache(entry)) continue; - if (unlikely(!is_node_folio(folio))) { + if (unlikely(!f2fs_is_node_cache(entry))) { continue_unlock: - folio_unlock(folio); + f2fs_unlock_cache(entry); continue; } - if (!folio_test_dirty(folio)) { + if (!f2fs_cache_test_dirty(entry)) { /* someone wrote it for us */ goto continue_unlock; } @@ -2182,38 +2194,40 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, goto write_node; /* flush inline_data */ - if (folio_test_f2fs_inline(folio)) { - folio_clear_f2fs_inline(folio); - folio_unlock(folio); - flush_inline_data(sbi, ino_of_node(folio)); + if (f2fs_cache_test_inline(entry)) { + nid_t ino = ino_of_node(sbi, entry); + + f2fs_cache_clear_inline(entry); + f2fs_unlock_cache(entry); + flush_inline_data(sbi, ino); goto lock_node; } /* flush dirty inode */ - if (IS_INODE(folio) && flush_dirty_inode(folio)) + if (IS_INODE(sbi, entry) && flush_dirty_cache_inode(entry)) goto lock_node; write_node: - f2fs_folio_wait_writeback(folio, NODE, true, true); + f2fs_cache_wait_writeback(entry); - if (!folio_clear_dirty_for_io(folio)) + if (!f2fs_cache_test_and_clear_dirty(entry)) goto continue_unlock; - if (!__write_node_folio(folio, false, false, &submitted, - wbc, do_balance, io_type, NULL)) { - folio_batch_release(&fbatch); + if (!__write_node_cache(entry, false, false, &submitted, + sync, do_balance, io_type, NULL)) { + f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; } if (submitted) nwritten++; - if (--wbc->nr_to_write == 0) + if (nwritten >= nr_to_write) break; } - folio_batch_release(&fbatch); + f2fs_cache_gang_release(entries, nr); cond_resched(); - if (wbc->nr_to_write == 0) { + if (nwritten >= nr_to_write) { step = 2; break; } @@ -2221,7 +2235,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, if (step < 2) { if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && - wbc->sync_mode == WB_SYNC_NONE && step == 1) + !sync && step == 1) goto out; step++; goto next_step; @@ -2230,12 +2244,17 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, if (nwritten) f2fs_submit_merged_write(sbi, NODE); + trace_f2fs_write_caches(sbi, nr_to_write, nwritten, NODE); + + if (nr_written) + *nr_written = nwritten; + if (unlikely(f2fs_cp_error(sbi))) return -EIO; return ret; } -int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, +int f2fs_wait_on_node_caches_writeback(struct f2fs_sb_info *sbi, unsigned int seq_id) { struct fsync_node_entry *fn; @@ -2244,7 +2263,7 @@ int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, unsigned int cur_seq_id = 0; while (seq_id && cur_seq_id < seq_id) { - struct folio *folio; + struct f2fs_cached_block *entry; spin_lock_irqsave(&sbi->fsync_node_lock, flags); if (list_empty(head)) { @@ -2257,94 +2276,50 @@ int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, break; } cur_seq_id = fn->seq_id; - folio = fn->folio; - folio_get(folio); + entry = fn->entry; + f2fs_cache_get(entry); spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); - f2fs_folio_wait_writeback(folio, NODE, true, false); + f2fs_lock_cache(entry); + f2fs_cache_wait_writeback(entry); - folio_put(folio); + f2fs_put_cache(entry, true); } - return filemap_check_errors(NODE_MAPPING(sbi)); + return f2fs_cp_error(sbi) ? -EIO : 0; } -static int f2fs_write_node_pages(struct address_space *mapping, - struct writeback_control *wbc) +int f2fs_write_node_caches(struct f2fs_sb_info *sbi, long *nr_written) { - struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); struct blk_plug plug; - long diff; + long nr_to_write = LONG_MAX; if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) - goto skip_write; + return -EAGAIN; /* balancing f2fs's metadata in background */ f2fs_balance_fs_bg(sbi, true); - /* collect a number of dirty node pages and write together */ - if (wbc->sync_mode != WB_SYNC_ALL && - get_pages(sbi, F2FS_DIRTY_NODES) < - nr_pages_to_skip(sbi, NODE)) - goto skip_write; + /* collect a number of dirty node caches and write together */ + if (get_nr_caches(sbi, F2FS_DIRTY_NODES) < + nr_caches_to_skip(sbi, NODE)) + return -EAGAIN; - if (wbc->sync_mode == WB_SYNC_ALL) - atomic_inc(&sbi->wb_sync_req[NODE]); - else if (atomic_read(&sbi->wb_sync_req[NODE])) { + if (atomic_read(&sbi->wb_sync_req[NODE])) { /* to avoid potential deadlock */ if (current->plug) blk_finish_plug(current->plug); - goto skip_write; + return -EAGAIN; } - trace_f2fs_writepages(mapping->host, wbc, NODE); - - diff = nr_pages_to_write(sbi, NODE, wbc); + nr_to_write = adjust_flush_cache_number(sbi, NODE); blk_start_plug(&plug); - f2fs_sync_node_pages(sbi, wbc, true, FS_NODE_IO); + f2fs_writeback_node_caches(sbi, nr_to_write, false, true, FS_NODE_IO, + nr_written); blk_finish_plug(&plug); - wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); - - if (wbc->sync_mode == WB_SYNC_ALL) - atomic_dec(&sbi->wb_sync_req[NODE]); - return 0; - -skip_write: - wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES); - trace_f2fs_writepages(mapping->host, wbc, NODE); return 0; } -static bool f2fs_dirty_node_folio(struct address_space *mapping, - struct folio *folio) -{ - trace_f2fs_set_page_dirty(folio, NODE); - - if (!folio_test_uptodate(folio)) - folio_mark_uptodate(folio); -#ifdef CONFIG_F2FS_CHECK_FS - if (IS_INODE(folio)) - f2fs_inode_chksum_set(F2FS_M_SB(mapping), folio); -#endif - if (filemap_dirty_folio(mapping, folio)) { - inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); - folio_set_f2fs_reference(folio); - return true; - } - return false; -} - -/* - * Structure of the f2fs node operations - */ -const struct address_space_operations f2fs_node_aops = { - .writepages = f2fs_write_node_pages, - .dirty_folio = f2fs_dirty_node_folio, - .invalidate_folio = f2fs_invalidate_folio, - .release_folio = f2fs_release_folio, - .migrate_folio = filemap_migrate_folio, -}; - static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i, nid_t n) { @@ -2403,8 +2378,8 @@ static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid, bool set, bool build) { struct f2fs_nm_info *nm_i = NM_I(sbi); - unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid); - unsigned int nid_ofs = nid - START_NID(nid); + unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, nid); + unsigned int nid_ofs = nid - f2fs_start_nid(sbi, nid); if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap)) return; @@ -2461,7 +2436,7 @@ static bool add_free_nid(struct f2fs_sb_info *sbi, * - f2fs_balance_fs_bg * - f2fs_build_free_nids * - __f2fs_build_free_nids - * - scan_nat_page + * - scan_nat_blocks * - add_free_nid * - __lookup_nat_cache * - f2fs_add_link @@ -2519,19 +2494,19 @@ static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid) kmem_cache_free(free_nid_slab, i); } -static int scan_nat_page(struct f2fs_sb_info *sbi, +static int scan_nat_block(struct f2fs_sb_info *sbi, struct f2fs_nat_block *nat_blk, nid_t start_nid) { struct f2fs_nm_info *nm_i = NM_I(sbi); block_t blk_addr; - unsigned int nat_ofs = NAT_BLOCK_OFFSET(start_nid); + unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, start_nid); int i; __set_bit_le(nat_ofs, nm_i->nat_block_bitmap); - i = start_nid % NAT_ENTRY_PER_BLOCK; + i = start_nid % NAT_ENTRY_PER_BLOCK(sbi); - for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) { + for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++, start_nid++) { if (unlikely(start_nid >= nm_i->max_nid)) break; @@ -2587,16 +2562,16 @@ static void scan_free_nid_bits(struct f2fs_sb_info *sbi) continue; if (!nm_i->free_nid_count[i]) continue; - for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) { + for (idx = 0; idx < NAT_ENTRY_PER_BLOCK(sbi); idx++) { idx = find_next_bit_le(nm_i->free_nid_bitmap[i], - NAT_ENTRY_PER_BLOCK, idx); - if (idx >= NAT_ENTRY_PER_BLOCK) + NAT_ENTRY_PER_BLOCK(sbi), idx); + if (idx >= NAT_ENTRY_PER_BLOCK(sbi)) break; - nid = i * NAT_ENTRY_PER_BLOCK + idx; + nid = i * NAT_ENTRY_PER_BLOCK(sbi) + idx; add_free_nid(sbi, nid, true, false); - if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS) + if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS(sbi)) goto out; } } @@ -2610,6 +2585,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount) { struct f2fs_nm_info *nm_i = NM_I(sbi); + struct f2fs_cached_block *entry = NULL; int i = 0, ret; nid_t nid = nm_i->next_scan_nid; struct f2fs_lock_context lc; @@ -2617,11 +2593,11 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, if (unlikely(nid >= nm_i->max_nid)) nid = 0; - if (unlikely(nid % NAT_ENTRY_PER_BLOCK)) - nid = NAT_BLOCK_OFFSET(nid) * NAT_ENTRY_PER_BLOCK; + if (unlikely(nid % NAT_ENTRY_PER_BLOCK(sbi))) + nid = NAT_BLOCK_OFFSET(sbi, nid) * NAT_ENTRY_PER_BLOCK(sbi); /* Enough entries */ - if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK) + if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi)) return 0; if (!sync && !f2fs_available_free_memory(sbi, FREE_NIDS)) @@ -2631,27 +2607,27 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, /* try to find free nids in free_nid_bitmap */ scan_free_nid_bits(sbi); - if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK) + if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi)) return 0; } - /* readahead nat pages to be scanned */ - f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, + /* readahead nat blocks to be scanned */ + f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), FREE_NID_BLOCKS, META_NAT, true); f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); while (1) { - if (!test_bit_le(NAT_BLOCK_OFFSET(nid), + if (!test_bit_le(NAT_BLOCK_OFFSET(sbi, nid), nm_i->nat_block_bitmap)) { - struct folio *folio = get_current_nat_folio(sbi, nid); + entry = get_current_nat_cache(sbi, nid); - if (IS_ERR(folio)) { - ret = PTR_ERR(folio); + if (IS_ERR(entry)) { + ret = PTR_ERR(entry); } else { - ret = scan_nat_page(sbi, folio_address(folio), + ret = scan_nat_block(sbi, cache_address(entry), nid); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } if (ret) { @@ -2668,24 +2644,25 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, } } - nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK)); + nid += NAT_ENTRY_PER_BLOCK(sbi) - + (nid % NAT_ENTRY_PER_BLOCK(sbi)); if (unlikely(nid >= nm_i->max_nid)) nid = 0; - if (++i >= FREE_NID_PAGES) + if (++i >= FREE_NID_BLOCKS) break; } - /* go to the next free nat pages to find free nids abundantly */ + /* go to the next free nat blocks to find free nids abundantly */ nm_i->next_scan_nid = nid; - /* find free nids from current sum_pages */ + /* find free nids from current sum_blocks */ scan_curseg_cache(sbi); f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); - f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid), - nm_i->ra_nid_pages, META_NAT, false); + f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nm_i->next_scan_nid), + nm_i->ra_nid_blocks, META_NAT, false); return 0; } @@ -2750,7 +2727,7 @@ bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid) } spin_unlock(&nm_i->nid_list_lock); - /* Let's scan nat pages and its caches to get free nids */ + /* Let's scan nat blocks and its caches to get free nids */ if (!f2fs_build_free_nids(sbi, true, false)) goto retry; return false; @@ -2811,20 +2788,20 @@ int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink) struct f2fs_nm_info *nm_i = NM_I(sbi); int nr = nr_shrink; - if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS) + if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi)) return 0; if (!mutex_trylock(&nm_i->build_lock)) return 0; - while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS) { + while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS(sbi)) { struct free_nid *i, *next; unsigned int batch = SHRINK_NID_BATCH_SIZE; spin_lock(&nm_i->nid_list_lock); list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) { if (!nr_shrink || !batch || - nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS) + nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi)) break; __remove_free_nid(sbi, i, FREE_NID); kmem_cache_free(free_nid_slab, i); @@ -2839,18 +2816,18 @@ int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink) return nr - nr_shrink; } -int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio) +int f2fs_recover_inline_xattr(struct inode *inode, struct f2fs_cached_block *entry) { void *src_addr, *dst_addr; size_t inline_size; - struct folio *ifolio; + struct f2fs_cached_block *ientry; struct f2fs_inode *ri; - ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); - if (IS_ERR(ifolio)) - return PTR_ERR(ifolio); + ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ientry)) + return PTR_ERR(ientry); - ri = F2FS_INODE(folio); + ri = &CACHED_NODE(entry)->i; if (ri->i_inline & F2FS_INLINE_XATTR) { if (!f2fs_has_inline_xattr(inode)) { set_inode_flag(inode, FI_INLINE_XATTR); @@ -2864,26 +2841,63 @@ int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio) goto update_inode; } - dst_addr = inline_xattr_addr(inode, ifolio); - src_addr = inline_xattr_addr(inode, folio); + dst_addr = inline_xattr_addr(inode, ientry); + src_addr = inline_xattr_addr(inode, entry); inline_size = inline_xattr_size(inode); - f2fs_folio_wait_writeback(ifolio, NODE, true, true); + f2fs_cache_wait_writeback(ientry); memcpy(dst_addr, src_addr, inline_size); update_inode: - f2fs_update_inode(inode, ifolio); - f2fs_folio_put(ifolio, true); + /* + * The xattr node of the checkpoint was removed before fsync, e.g. the + * xattrs became small enough to fit inline. Release it as well, or + * the inode keeps pointing to the stale xattrs. + */ + if (!ri->i_xattr_nid && F2FS_I(inode)->i_xattr_nid) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct node_info ni; + int err; + + err = f2fs_get_node_info(sbi, F2FS_I(inode)->i_xattr_nid, + &ni, false); + if (err) { + f2fs_put_cache(ientry, true); + return err; + } + + if (ni.blk_addr != NEW_ADDR && + !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, + DATA_GENERIC_ENHANCE)) { + f2fs_err_ratelimited(sbi, + "nat entry is corrupted, run fsck to fix it, ino:%u, nid:%u, blkaddr:%u", + ni.ino, ni.nid, ni.blk_addr); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + f2fs_put_cache(ientry, true); + return -EFSCORRUPTED; + } + + f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); + dec_valid_node_count(sbi, inode, false); + set_node_addr(sbi, &ni, NULL_ADDR, false); + /* drop the cached xattr node, which may be dirty */ + f2fs_truncate_node_caches(sbi, ni.nid, 1); + f2fs_i_xnid_write(inode, 0); + } + + f2fs_update_inode(inode, ientry); + f2fs_put_cache(ientry, true); return 0; } -int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio) +int f2fs_recover_xattr_data(struct inode *inode, struct f2fs_cached_block *entry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid; nid_t new_xnid; struct dnode_of_data dn; struct node_info ni; - struct folio *xfolio; + struct f2fs_cached_block *xentry; int err; if (!prev_xnid) @@ -2904,32 +2918,33 @@ int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio) return -ENOSPC; set_new_dnode(&dn, inode, NULL, NULL, new_xnid); - xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET); - if (IS_ERR(xfolio)) { + xentry = f2fs_new_node_cache(&dn, XATTR_NODE_OFFSET); + if (IS_ERR(xentry)) { f2fs_alloc_nid_failed(sbi, new_xnid); - return PTR_ERR(xfolio); + return PTR_ERR(xentry); } f2fs_alloc_nid_done(sbi, new_xnid); - f2fs_update_inode_page(inode); + f2fs_update_inode_cache(inode); - /* 3: update and set xattr node page dirty */ - if (folio) { - memcpy(F2FS_NODE(xfolio), F2FS_NODE(folio), - VALID_XATTR_BLOCK_SIZE); - folio_mark_dirty(xfolio); + /* 3: update and set xattr node cache dirty */ + if (entry) { + memcpy(CACHED_NODE(xentry), CACHED_NODE(entry), + VALID_XATTR_BLOCK_SIZE(inode)); + f2fs_mark_cache_dirty(xentry); } - f2fs_folio_put(xfolio, true); + f2fs_put_cache(xentry, true); return 0; } -int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio) +int f2fs_recover_inode_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry) { struct f2fs_inode *src, *dst; - nid_t ino = ino_of_node(folio); + nid_t ino = ino_of_node(sbi, entry); struct node_info old_ni, new_ni; - struct folio *ifolio; + struct f2fs_cached_block *ientry; int err; err = f2fs_get_node_info(sbi, ino, &old_ni, false); @@ -2939,8 +2954,8 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio) if (unlikely(old_ni.blk_addr != NULL_ADDR)) return -EINVAL; retry: - ifolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), ino, false); - if (IS_ERR(ifolio)) { + ientry = f2fs_grab_node_cache(sbi, ino); + if (IS_ERR(ientry)) { memalloc_retry_wait(GFP_NOFS); goto retry; } @@ -2948,13 +2963,12 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio) /* Should not use this inode from free nid list */ remove_free_nid(sbi, ino); - if (!folio_test_uptodate(ifolio)) - folio_mark_uptodate(ifolio); - fill_node_footer(ifolio, ino, ino, 0, true); - set_cold_node(ifolio, false); + f2fs_cache_set_uptodate(ientry); + fill_node_footer(sbi, ientry, ino, ino, 0, true); + set_cold_node(sbi, ientry, false); - src = F2FS_INODE(folio); - dst = F2FS_INODE(ifolio); + src = &CACHED_NODE(entry)->i; + dst = F2FS_INODE(ientry); memcpy(dst, src, offsetof(struct f2fs_inode, i_ext)); dst->i_size = 0; @@ -2990,46 +3004,46 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio) WARN_ON(1); set_node_addr(sbi, &new_ni, NEW_ADDR, false); inc_valid_inode_count(sbi); - folio_mark_dirty(ifolio); - f2fs_folio_put(ifolio, true); + f2fs_mark_cache_dirty(ientry); + f2fs_put_cache(ientry, true); return 0; } int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, unsigned int segno, struct f2fs_summary_block *sum) { - struct f2fs_node *rn; struct f2fs_summary *sum_entry; block_t addr; - int i, idx, last_offset, nrpages; + int i, idx, last_offset, nrblocks; /* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); sum_entry = sum_entries(sum); - for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { - nrpages = bio_max_segs(last_offset - i); + for (i = 0; i < last_offset; i += nrblocks, addr += nrblocks) { + nrblocks = bio_max_segs(last_offset - i); - /* readahead node pages */ - f2fs_ra_meta_pages(sbi, addr, nrpages, META_POR, true); + /* readahead node blocks */ + f2fs_ra_meta_caches(sbi, addr, nrblocks, META_POR, true); - for (idx = addr; idx < addr + nrpages; idx++) { - struct folio *folio = f2fs_get_tmp_folio(sbi, idx); + for (idx = addr; idx < addr + nrblocks; idx++) { + struct f2fs_cached_block *entry = + f2fs_get_tmp_cache(sbi, idx); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(entry)) + return PTR_ERR(entry); - rn = F2FS_NODE(folio); - sum_entry->nid = rn->footer.nid; + struct node_footer *footer = (struct node_footer *)(cache_address(entry) + + F2FS_BLKSIZE(sbi) - sizeof(struct node_footer)); + sum_entry->nid = footer->nid; sum_entry->version = 0; sum_entry->ofs_in_node = 0; sum_entry++; - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } - invalidate_mapping_pages(META_MAPPING(sbi), addr, - addr + nrpages); + f2fs_truncate_meta_caches(sbi, addr, nrblocks); } return 0; } @@ -3074,7 +3088,7 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi) spin_unlock(&nm_i->nid_list_lock); } - __set_nat_cache_dirty(nm_i, ne, init_dirty); + __set_nat_cache_dirty(sbi, nm_i, ne, init_dirty); } update_nats_in_cursum(journal, -i); up_write(&curseg->journal_rwsem); @@ -3102,7 +3116,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid, const struct f2fs_nat_block *nat_blk) { struct f2fs_nm_info *nm_i = NM_I(sbi); - unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK; + unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK(sbi); int valid = 0; int i = 0; @@ -3113,7 +3127,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid, valid = 1; i = 1; } - for (; i < NAT_ENTRY_PER_BLOCK; i++) { + for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++) { if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR) valid++; } @@ -3124,7 +3138,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid, } __clear_bit_le(nat_index, nm_i->empty_nat_bits); - if (valid == NAT_ENTRY_PER_BLOCK) + if (valid == NAT_ENTRY_PER_BLOCK(sbi)) __set_bit_le(nat_index, nm_i->full_nat_bits); else __clear_bit_le(nat_index, nm_i->full_nat_bits); @@ -3135,16 +3149,16 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, { struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; - nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK; + nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK(sbi); bool to_journal = true; struct f2fs_nat_block *nat_blk; struct nat_entry *ne, *cur; - struct folio *folio = NULL; + struct f2fs_cached_block *entry = NULL; /* * there are two steps to flush nat entries: * #1, flush nat entries to journal in current hot data summary block. - * #2, flush nat entries to nat page. + * #2, flush nat entries to nat block. */ if (enabled_nat_bits(sbi, cpc) || !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) @@ -3153,11 +3167,11 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, if (to_journal) { down_write(&curseg->journal_rwsem); } else { - folio = get_next_nat_folio(sbi, start_nid); - if (IS_ERR(folio)) - return PTR_ERR(folio); + entry = get_next_nat_cache(sbi, start_nid); + if (IS_ERR(entry)) + return PTR_ERR(entry); - nat_blk = folio_address(folio); + nat_blk = cache_address(entry); f2fs_bug_on(sbi, !nat_blk); } @@ -3194,7 +3208,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, up_write(&curseg->journal_rwsem); } else { __update_nat_bits(sbi, start_nid, nat_blk); - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); } /* Allow dirty nats by node block allocation in write_begin */ @@ -3266,7 +3280,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) __has_cursum_space(sbi, journal, entry_count, NAT_JOURNAL)) continue; - f2fs_ra_meta_pages(sbi, set->set, 1, META_NAT, true); + f2fs_ra_meta_caches(sbi, set->set, 1, META_NAT, true); } /* flush dirty nats in nat entry set */ list_for_each_entry_safe(set, tmp, &sets, set_list) { @@ -3293,24 +3307,25 @@ static int __get_nat_bitmaps(struct f2fs_sb_info *sbi) if (!enabled_nat_bits(sbi, NULL)) return 0; - nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8); + nm_i->nat_bits_blocks = F2FS_BLK_ALIGN(sbi, + (nat_bits_bytes << 1) + 8); nm_i->nat_bits = f2fs_kvzalloc(sbi, - F2FS_BLK_TO_BYTES(nm_i->nat_bits_blocks), GFP_KERNEL); + F2FS_BLK_TO_BYTES(sbi, nm_i->nat_bits_blocks), GFP_KERNEL); if (!nm_i->nat_bits) return -ENOMEM; nat_bits_addr = __start_cp_addr(sbi) + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks; for (i = 0; i < nm_i->nat_bits_blocks; i++) { - struct folio *folio; + struct f2fs_cached_block *entry; - folio = f2fs_get_meta_folio(sbi, nat_bits_addr++); - if (IS_ERR(folio)) - return PTR_ERR(folio); + entry = f2fs_get_meta_cache(sbi, nat_bits_addr++); + if (IS_ERR(entry)) + return PTR_ERR(entry); - memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(i), - folio_address(folio), F2FS_BLKSIZE); - f2fs_folio_put(folio, true); + memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(sbi, i), + cache_address(entry), F2FS_BLKSIZE(sbi)); + f2fs_put_cache(entry, true); } cp_ver |= (cur_cp_crc(ckpt) << 32); @@ -3342,8 +3357,8 @@ static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi) __set_bit_le(i, nm_i->nat_block_bitmap); - nid = i * NAT_ENTRY_PER_BLOCK; - last_nid = nid + NAT_ENTRY_PER_BLOCK; + nid = i * NAT_ENTRY_PER_BLOCK(sbi); + last_nid = nid + NAT_ENTRY_PER_BLOCK(sbi); spin_lock(&NM_I(sbi)->nid_list_lock); for (; nid < last_nid; nid++) @@ -3373,7 +3388,7 @@ static int init_node_manager(struct f2fs_sb_info *sbi) /* segment_count_nat includes pair segment so divide to 2. */ nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1; nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg); - nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nm_i->nat_blocks; + nm_i->max_nid = NAT_ENTRY_PER_BLOCK(sbi) * nm_i->nat_blocks; /* not used nids: 0, node, meta, (and root counted as valid node) */ nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count - @@ -3381,7 +3396,7 @@ static int init_node_manager(struct f2fs_sb_info *sbi) nm_i->nid_cnt[FREE_NID] = 0; nm_i->nid_cnt[PREALLOC_NID] = 0; nm_i->ram_thresh = DEF_RAM_THRESHOLD; - nm_i->ra_nid_pages = DEF_RA_NID_PAGES; + nm_i->ra_nid_blocks = DEF_RA_NID_BLOCKS; nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD; nm_i->max_rf_node_blocks = DEF_RF_NODE_BLOCKS; @@ -3436,7 +3451,7 @@ static int init_free_nid_cache(struct f2fs_sb_info *sbi) for (i = 0; i < nm_i->nat_blocks; i++) { nm_i->free_nid_bitmap[i] = f2fs_kvzalloc(sbi, - f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK), GFP_KERNEL); + f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK(sbi)), GFP_KERNEL); if (!nm_i->free_nid_bitmap[i]) return -ENOMEM; } diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 5e114f352099..ff1f04864391 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -6,19 +6,26 @@ * http://www.samsung.com/ */ /* start node id of a node block dedicated to the given node id */ -#define START_NID(nid) (((nid) / NAT_ENTRY_PER_BLOCK) * NAT_ENTRY_PER_BLOCK) +static inline nid_t f2fs_start_nid(struct f2fs_sb_info *sbi, nid_t nid) +{ + unsigned int entries = NAT_ENTRY_PER_BLOCK(sbi); + + return (nid / entries) * entries; +} /* node block offset on the NAT area dedicated to the given start node id */ -#define NAT_BLOCK_OFFSET(start_nid) ((start_nid) / NAT_ENTRY_PER_BLOCK) +#define NAT_BLOCK_OFFSET(sbi, start_nid) \ + ((start_nid) / NAT_ENTRY_PER_BLOCK(sbi)) -/* # of pages to perform synchronous readahead before building free nids */ -#define FREE_NID_PAGES 8 -#define MAX_FREE_NIDS (NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES) +/* # of blocks to perform synchronous readahead before building free nids */ +#define FREE_NID_BLOCKS 8 +#define MAX_FREE_NIDS(sbi) ((unsigned long)NAT_ENTRY_PER_BLOCK(sbi) * \ + FREE_NID_BLOCKS) /* size of free nid batch when shrinking */ #define SHRINK_NID_BATCH_SIZE 8 -#define DEF_RA_NID_PAGES 0 /* # of nid pages to be readaheaded */ +#define DEF_RA_NID_BLOCKS 0 /* # of nid blocks to be readaheaded */ /* maximum readahead size for node during getting data blocks */ #define MAX_RA_NODE 128 @@ -37,8 +44,8 @@ /* vector size for gang look-up from nat cache that consists of radix tree */ #define NAT_VEC_SIZE 32 -/* return value for read_node_page */ -#define LOCKED_PAGE 1 +/* return value for read_node_cache */ +#define LOCKED_CACHE 1 /* check pinned file's alignment status of physical blocks */ #define FILE_NOT_ALIGNED 1 @@ -150,7 +157,9 @@ enum mem_type { READ_EXTENT_CACHE, /* indicates read extent cache */ AGE_EXTENT_CACHE, /* indicates age extent cache */ DISCARD_CACHE, /* indicates memory of cached discard cmds */ - COMPRESS_PAGE, /* indicates memory of cached compressed pages */ + COMPRESS_BLOCK, /* indicates memory of cached compressed blocks */ + META_BLOCK, /* indicates memory of cached meta blocks */ + NODE_BLOCK, /* indicates memory of cached node blocks */ BASE_CHECK, /* check kernel status */ }; @@ -208,7 +217,7 @@ static inline pgoff_t current_nat_addr(struct f2fs_sb_info *sbi, nid_t start) * OLD = (segment_off * 512) * 2 + off_in_segment * NEW = 2 * (segment_off * 512 + off_in_segment) - off_in_segment */ - block_off = NAT_BLOCK_OFFSET(start); + block_off = NAT_BLOCK_OFFSET(sbi, start); block_addr = (pgoff_t)(nm_i->nat_blkaddr + (block_off << 1) - @@ -230,9 +239,10 @@ static inline pgoff_t next_nat_addr(struct f2fs_sb_info *sbi, return block_addr + nm_i->nat_blkaddr; } -static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid) +static inline void set_to_next_nat(struct f2fs_sb_info *sbi, + struct f2fs_nm_info *nm_i, nid_t start_nid) { - unsigned int block_off = NAT_BLOCK_OFFSET(start_nid); + unsigned int block_off = NAT_BLOCK_OFFSET(sbi, start_nid); f2fs_change_bit(block_off, nm_i->nat_bitmap); #ifdef CONFIG_F2FS_CHECK_FS @@ -240,90 +250,95 @@ static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid) #endif } -static inline nid_t ino_of_node(const struct folio *node_folio) +static inline nid_t ino_of_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_node *rn = F2FS_NODE(node_folio); - return le32_to_cpu(rn->footer.ino); + return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->ino); } -static inline nid_t nid_of_node(const struct folio *node_folio) +static inline nid_t nid_of_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_node *rn = F2FS_NODE(node_folio); - return le32_to_cpu(rn->footer.nid); + return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->nid); } -static inline unsigned int ofs_of_node(const struct folio *node_folio) +static inline unsigned int ofs_of_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_node *rn = F2FS_NODE(node_folio); - unsigned flag = le32_to_cpu(rn->footer.flag); + unsigned int flag = le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->flag); return flag >> OFFSET_BIT_SHIFT; } -static inline __u64 cpver_of_node(const struct folio *node_folio) +static inline __u64 cpver_of_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_node *rn = F2FS_NODE(node_folio); - return le64_to_cpu(rn->footer.cp_ver); + return le64_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->cp_ver); } -static inline block_t next_blkaddr_of_node(const struct folio *node_folio) +static inline block_t next_blkaddr_of_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_node *rn = F2FS_NODE(node_folio); - return le32_to_cpu(rn->footer.next_blkaddr); + return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->next_blkaddr); } -static inline void fill_node_footer(const struct folio *folio, nid_t nid, +static inline void fill_node_footer(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, nid_t nid, nid_t ino, unsigned int ofs, bool reset) { - struct f2fs_node *rn = F2FS_NODE(folio); + struct f2fs_node *rn = CACHED_NODE(entry); + struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry); unsigned int old_flag = 0; if (reset) - memset(rn, 0, sizeof(*rn)); + memset(rn, 0, F2FS_BLKSIZE(sbi)); else - old_flag = le32_to_cpu(rn->footer.flag); + old_flag = le32_to_cpu(footer->flag); - rn->footer.nid = cpu_to_le32(nid); - rn->footer.ino = cpu_to_le32(ino); + memset(footer, 0, sizeof(*footer)); + footer->nid = cpu_to_le32(nid); + footer->ino = cpu_to_le32(ino); /* should remain old flag bits such as COLD_BIT_SHIFT */ - rn->footer.flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) | + footer->flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) | (old_flag & OFFSET_BIT_MASK)); } -static inline void copy_node_footer(const struct folio *dst, - const struct folio *src) +static inline void copy_node_footer(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *dst, + const struct f2fs_cached_block *src) { - struct f2fs_node *src_rn = F2FS_NODE(src); - struct f2fs_node *dst_rn = F2FS_NODE(dst); - memcpy(&dst_rn->footer, &src_rn->footer, sizeof(struct node_footer)); + memcpy(F2FS_NODE_FOOTER(sbi, dst), F2FS_NODE_FOOTER(sbi, src), + sizeof(struct node_footer)); } -static inline void fill_node_footer_blkaddr(struct folio *folio, block_t blkaddr) +static inline void fill_node_footer_blkaddr(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, block_t blkaddr) { - struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio)); - struct f2fs_node *rn = F2FS_NODE(folio); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry); __u64 cp_ver = cur_cp_version(ckpt); if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG)) cp_ver |= (cur_cp_crc(ckpt) << 32); - rn->footer.cp_ver = cpu_to_le64(cp_ver); - rn->footer.next_blkaddr = cpu_to_le32(blkaddr); + footer->cp_ver = cpu_to_le64(cp_ver); + footer->next_blkaddr = cpu_to_le32(blkaddr); } -static inline bool is_recoverable_dnode(const struct folio *folio) +static inline bool is_recoverable_dnode(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio)); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); __u64 cp_ver = cur_cp_version(ckpt); /* Don't care crc part, if fsck.f2fs sets it. */ if (__is_set_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG)) - return (cp_ver << 32) == (cpver_of_node(folio) << 32); + return (cp_ver << 32) == (cpver_of_node(sbi, entry) << 32); if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG)) cp_ver |= (cur_cp_crc(ckpt) << 32); - return cp_ver == cpver_of_node(folio); + return cp_ver == cpver_of_node(sbi, entry); } /* @@ -347,43 +362,50 @@ static inline bool is_recoverable_dnode(const struct folio *folio) * `- indirect node ((6 + 2N) + (N - 1)(N + 1)) * `- direct node */ -static inline bool IS_DNODE(const struct folio *node_folio) +static inline bool IS_DNODE(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry) { - unsigned int ofs = ofs_of_node(node_folio); + unsigned int ofs = ofs_of_node(sbi, entry); if (f2fs_has_xattr_block(ofs)) return true; - if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK || - ofs == 5 + 2 * NIDS_PER_BLOCK) + if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK(sbi) || + ofs == 5 + 2 * NIDS_PER_BLOCK(sbi)) return false; - if (ofs >= 6 + 2 * NIDS_PER_BLOCK) { - ofs -= 6 + 2 * NIDS_PER_BLOCK; - if (!((long int)ofs % (NIDS_PER_BLOCK + 1))) + if (ofs >= 6 + 2 * NIDS_PER_BLOCK(sbi)) { + ofs -= 6 + 2 * NIDS_PER_BLOCK(sbi); + if (!((long)ofs % (NIDS_PER_BLOCK(sbi) + 1))) return false; } return true; } -static inline int set_nid(struct folio *folio, int off, nid_t nid, bool i) +static inline bool set_nid(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, int off, nid_t nid, bool i) { - struct f2fs_node *rn = F2FS_NODE(folio); + struct f2fs_node *rn = CACHED_NODE(entry); + __le32 *inode_nids = F2FS_INODE_NIDS(sbi, entry); + __le32 *addr = i ? &inode_nids[off - NODE_DIR1_BLOCK(sbi)] : &rn->in.nid[off]; - f2fs_folio_wait_writeback(folio, NODE, true, true); + f2fs_cache_wait_writeback(entry); + if (*addr == cpu_to_le32(nid)) + return false; - if (i) - rn->i.i_nid[off - NODE_DIR1_BLOCK] = cpu_to_le32(nid); - else - rn->in.nid[off] = cpu_to_le32(nid); - return folio_mark_dirty(folio); + *addr = cpu_to_le32(nid); + f2fs_mark_cache_dirty(entry); + return true; } -static inline nid_t get_nid(const struct folio *folio, int off, bool i) +static inline nid_t get_nid(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry, int off, bool i) { - struct f2fs_node *rn = F2FS_NODE(folio); + struct f2fs_node *rn = CACHED_NODE(entry); + const __le32 *inode_nids = F2FS_INODE_NIDS(sbi, entry); + int nid_index = off - NODE_DIR1_BLOCK(sbi); if (i) - return le32_to_cpu(rn->i.i_nid[off - NODE_DIR1_BLOCK]); + return le32_to_cpu(inode_nids[nid_index]); return le32_to_cpu(rn->in.nid[off]); } @@ -394,40 +416,43 @@ static inline nid_t get_nid(const struct folio *folio, int off, bool i) * - Mark cold data pages in page cache */ -static inline int is_node(const struct folio *folio, int type) +static inline int is_node(struct f2fs_sb_info *sbi, + const struct f2fs_cached_block *entry, int type) { - struct f2fs_node *rn = F2FS_NODE(folio); - return le32_to_cpu(rn->footer.flag) & BIT(type); + return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->flag) & BIT(type); } -#define is_cold_node(folio) is_node(folio, COLD_BIT_SHIFT) -#define is_fsync_dnode(folio) is_node(folio, FSYNC_BIT_SHIFT) -#define is_dent_dnode(folio) is_node(folio, DENT_BIT_SHIFT) +#define is_cold_node(sbi, entry) is_node(sbi, entry, COLD_BIT_SHIFT) +#define is_fsync_dnode(sbi, entry) is_node(sbi, entry, FSYNC_BIT_SHIFT) +#define is_dent_dnode(sbi, entry) is_node(sbi, entry, DENT_BIT_SHIFT) -static inline void __set_mark(const struct folio *folio, bool mark, int type) +static inline void __set_mark(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, bool mark, int type) { - struct f2fs_node *rn = F2FS_NODE(folio); - unsigned int flag = le32_to_cpu(rn->footer.flag); + struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry); + unsigned int flag = le32_to_cpu(footer->flag); if (mark) flag |= BIT(type); else flag &= ~BIT(type); - rn->footer.flag = cpu_to_le32(flag); + footer->flag = cpu_to_le32(flag); } -static inline void set_cold_node(const struct folio *folio, bool is_dir) +static inline void set_cold_node(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, bool is_dir) { - __set_mark(folio, !is_dir, COLD_BIT_SHIFT); + __set_mark(sbi, entry, !is_dir, COLD_BIT_SHIFT); } -static inline void set_mark(struct folio *folio, bool mark, int type) +static inline void set_mark(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, bool mark, int type) { - __set_mark(folio, mark, type); - + __set_mark(sbi, entry, mark, type); #ifdef CONFIG_F2FS_CHECK_FS - f2fs_inode_chksum_set(F2FS_F_SB(folio), folio); + f2fs_inode_chksum_set(sbi, entry); #endif } -#define set_dentry_mark(folio, mark) set_mark(folio, mark, DENT_BIT_SHIFT) -#define set_fsync_mark(folio, mark) set_mark(folio, mark, FSYNC_BIT_SHIFT) + +#define set_dentry_mark(sbi, entry, mark) set_mark(sbi, entry, mark, DENT_BIT_SHIFT) +#define set_fsync_mark(sbi, entry, mark) set_mark(sbi, entry, mark, FSYNC_BIT_SHIFT) diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index aaa5227739c8..54b5246609de 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -182,38 +182,39 @@ static const char *recover_printable_name(struct inode *inode, return raw->i_name; } -static int recover_dentry(struct inode *inode, struct folio *ifolio, +static int recover_dentry(struct inode *inode, struct f2fs_cached_block *entry, struct list_head *dir_list) { - struct f2fs_inode *raw_inode = F2FS_INODE(ifolio); + struct f2fs_inode *raw_inode = &CACHED_NODE(entry)->i; nid_t pino = le32_to_cpu(raw_inode->i_pino); struct f2fs_dir_entry *de; struct f2fs_filename fname; struct qstr usr_fname; - struct folio *folio; + void *dentry_blk = NULL; struct inode *dir, *einode; - struct fsync_inode_entry *entry; + struct fsync_inode_entry *fsync_entry; int err = 0; const char *name; int name_len; - entry = get_fsync_inode(dir_list, pino); - if (!entry) { - entry = add_fsync_inode(F2FS_I_SB(inode), dir_list, + fsync_entry = get_fsync_inode(dir_list, pino); + if (!fsync_entry) { + fsync_entry = add_fsync_inode(F2FS_I_SB(inode), dir_list, pino, false); - if (IS_ERR(entry)) { - dir = ERR_CAST(entry); - err = PTR_ERR(entry); + if (IS_ERR(fsync_entry)) { + dir = ERR_CAST(fsync_entry); + err = PTR_ERR(fsync_entry); goto out; } } - dir = entry->inode; + dir = fsync_entry->inode; err = init_recovered_filename(dir, inode, raw_inode, &fname, &usr_fname); if (err) goto out; retry: - de = __f2fs_find_entry(dir, &fname, &folio); + dentry_blk = NULL; + de = __f2fs_find_entry(dir, &fname, &dentry_blk); if (de && inode->i_ino == le32_to_cpu(de->ino)) goto out_put; @@ -238,11 +239,11 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, iput(einode); goto out_put; } - f2fs_delete_entry(de, folio, dir, einode); + f2fs_delete_entry(de, dentry_blk, dir, einode); iput(einode); goto retry; - } else if (IS_ERR(folio)) { - err = PTR_ERR(folio); + } else if (IS_ERR(dentry_blk)) { + err = PTR_ERR(dentry_blk); } else { err = f2fs_add_dentry(dir, &fname, inode, inode->i_ino, inode->i_mode); @@ -252,18 +253,18 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, goto out; out_put: - f2fs_folio_put(folio, false); + f2fs_put_dentry_block(dentry_blk, false); out: name = recover_printable_name(inode, raw_inode, &name_len); f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, dir = %llu, err = %d", - __func__, ino_of_node(ifolio), name_len, name, + __func__, ino_of_node(F2FS_I_SB(inode), entry), name_len, name, IS_ERR(dir) ? 0 : dir->i_ino, err); return err; } -static int recover_quota_data(struct inode *inode, struct folio *folio) +static int recover_quota_data(struct inode *inode, struct f2fs_cached_block *entry) { - struct f2fs_inode *raw = F2FS_INODE(folio); + struct f2fs_inode *raw = &CACHED_NODE(entry)->i; struct iattr attr; uid_t i_uid = le32_to_cpu(raw->i_uid); gid_t i_gid = le32_to_cpu(raw->i_gid); @@ -300,9 +301,9 @@ static void recover_inline_flags(struct inode *inode, struct f2fs_inode *ri) clear_inode_flag(inode, FI_DATA_EXIST); } -static int recover_inode(struct inode *inode, struct folio *folio) +static int recover_inode(struct inode *inode, struct f2fs_cached_block *entry) { - struct f2fs_inode *raw = F2FS_INODE(folio); + struct f2fs_inode *raw = &CACHED_NODE(entry)->i; struct f2fs_inode_info *fi = F2FS_I(inode); const char *name; int name_len; @@ -310,7 +311,7 @@ static int recover_inode(struct inode *inode, struct folio *folio) inode->i_mode = le16_to_cpu(raw->i_mode); - err = recover_quota_data(inode, folio); + err = recover_quota_data(inode, entry); if (err) return err; @@ -357,7 +358,7 @@ static int recover_inode(struct inode *inode, struct folio *folio) name = recover_printable_name(inode, raw, &name_len); f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, inline = %x", - __func__, ino_of_node(folio), name_len, name, + __func__, ino_of_node(F2FS_I_SB(inode), entry), name_len, name, raw->i_inline); return 0; } @@ -386,30 +387,30 @@ static int sanity_check_node_chain(struct f2fs_sb_info *sbi, block_t blkaddr, return 0; for (i = 0; i < 2; i++) { - struct folio *folio; + struct f2fs_cached_block *entry; if (!f2fs_is_valid_blkaddr(sbi, *blkaddr_fast, META_POR)) { *is_detecting = false; return 0; } - folio = f2fs_get_tmp_folio(sbi, *blkaddr_fast); - if (IS_ERR(folio)) - return PTR_ERR(folio); + entry = f2fs_get_tmp_cache(sbi, *blkaddr_fast); + if (IS_ERR(entry)) + return PTR_ERR(entry); - if (!is_recoverable_dnode(folio)) { - f2fs_folio_put(folio, true); + if (!is_recoverable_dnode(sbi, entry)) { + f2fs_put_cache(entry, true); *is_detecting = false; return 0; } ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, *blkaddr_fast, - next_blkaddr_of_node(folio)); + next_blkaddr_of_node(sbi, entry)); - *blkaddr_fast = next_blkaddr_of_node(folio); - f2fs_folio_put(folio, true); + *blkaddr_fast = next_blkaddr_of_node(sbi, entry); + f2fs_put_cache(entry, true); - f2fs_ra_meta_pages_cond(sbi, *blkaddr_fast, ra_blocks); + f2fs_ra_meta_caches_cond(sbi, *blkaddr_fast, ra_blocks); } if (*blkaddr_fast == blkaddr) { @@ -434,45 +435,45 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head, blkaddr_fast = blkaddr; while (1) { - struct fsync_inode_entry *entry; - struct folio *folio; + struct fsync_inode_entry *fsync_entry; + struct f2fs_cached_block *entry; if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR)) return 0; - folio = f2fs_get_tmp_folio(sbi, blkaddr); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + entry = f2fs_get_tmp_cache(sbi, blkaddr); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); break; } - if (!is_recoverable_dnode(folio)) { - f2fs_folio_put(folio, true); + if (!is_recoverable_dnode(sbi, entry)) { + f2fs_put_cache(entry, true); break; } - if (!is_fsync_dnode(folio)) + if (!is_fsync_dnode(sbi, entry)) goto next; - entry = get_fsync_inode(head, ino_of_node(folio)); - if (!entry) { + fsync_entry = get_fsync_inode(head, ino_of_node(sbi, entry)); + if (!fsync_entry) { bool quota_inode = false; if (!check_only && - IS_INODE(folio) && - is_dent_dnode(folio)) { - err = f2fs_recover_inode_page(sbi, folio); + IS_INODE(sbi, entry) && + is_dent_dnode(sbi, entry)) { + err = f2fs_recover_inode_cache(sbi, entry); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); break; } quota_inode = true; } - entry = add_fsync_inode(sbi, head, ino_of_node(folio), + fsync_entry = add_fsync_inode(sbi, head, ino_of_node(sbi, entry), quota_inode); - if (IS_ERR(entry)) { - err = PTR_ERR(entry); + if (IS_ERR(fsync_entry)) { + err = PTR_ERR(fsync_entry); /* * CP | dnode(F) | inode(DF) * For this case, we should not give up now. @@ -482,18 +483,18 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head, *new_inode = true; goto next; } - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); break; } } - entry->blkaddr = blkaddr; + fsync_entry->blkaddr = blkaddr; - if (IS_INODE(folio) && is_dent_dnode(folio)) - entry->last_dentry = blkaddr; + if (IS_INODE(sbi, entry) && is_dent_dnode(sbi, entry)) + fsync_entry->last_dentry = blkaddr; next: /* check next segment */ - blkaddr = next_blkaddr_of_node(folio); - f2fs_folio_put(folio, true); + blkaddr = next_blkaddr_of_node(sbi, entry); + f2fs_put_cache(entry, true); err = sanity_check_node_chain(sbi, blkaddr, &blkaddr_fast, &is_detecting); @@ -519,7 +520,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, unsigned short blkoff = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); struct f2fs_summary_block *sum_node; struct f2fs_summary sum; - struct folio *sum_folio, *node_folio; + struct f2fs_cached_block *entry = NULL; + struct f2fs_cached_block *node_entry; struct dnode_of_data tdn = *dn; nid_t ino, nid; struct inode *inode; @@ -541,18 +543,18 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, } } - sum_folio = f2fs_get_sum_folio(sbi, segno); - if (IS_ERR(sum_folio)) - return PTR_ERR(sum_folio); - sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + entry = f2fs_get_sum_cache(sbi, segno); + if (IS_ERR(entry)) + return PTR_ERR(entry); + sum_node = SUM_BLK_ENTRY_ADDR(sbi, entry, segno); sum = sum_entries(sum_node)[blkoff]; - f2fs_folio_put(sum_folio, true); + f2fs_put_cache(entry, true); got_it: /* Use the locked dnode page and inode */ nid = le32_to_cpu(sum.nid); ofs_in_node = le16_to_cpu(sum.ofs_in_node); - max_addrs = ADDRS_PER_PAGE(dn->node_folio, dn->inode); + max_addrs = ADDRS_PER_PAGE(dn->node_entry, dn->inode); if (ofs_in_node >= max_addrs) { f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%llu, nid:%u, max:%u", ofs_in_node, dn->inode->i_ino, nid, max_addrs); @@ -562,9 +564,9 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, if (dn->inode->i_ino == nid) { tdn.nid = nid; - if (!dn->inode_folio_locked) - folio_lock(dn->inode_folio); - tdn.node_folio = dn->inode_folio; + if (!dn->inode_entry_locked) + f2fs_lock_cache(dn->inode_entry); + tdn.node_entry = dn->inode_entry; tdn.ofs_in_node = ofs_in_node; goto truncate_out; } else if (dn->nid == nid) { @@ -573,13 +575,13 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, } /* Get the node page */ - node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); - if (IS_ERR(node_folio)) - return PTR_ERR(node_folio); + node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_entry)) + return PTR_ERR(node_entry); - offset = ofs_of_node(node_folio); - ino = ino_of_node(node_folio); - f2fs_folio_put(node_folio, true); + offset = ofs_of_node(sbi, node_entry); + ino = ino_of_node(sbi, node_entry); + f2fs_put_cache(node_entry, true); if (ino != dn->inode->i_ino) { int ret; @@ -605,8 +607,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, * if inode page is locked, unlock temporarily, but its reference * count keeps alive. */ - if (ino == dn->inode->i_ino && dn->inode_folio_locked) - folio_unlock(dn->inode_folio); + if (ino == dn->inode->i_ino && dn->inode_entry_locked) + f2fs_unlock_cache(dn->inode_entry); set_new_dnode(&tdn, inode, NULL, NULL, 0); if (f2fs_get_dnode_of_data(&tdn, bidx, LOOKUP_NODE)) @@ -619,15 +621,15 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, out: if (ino != dn->inode->i_ino) iput(inode); - else if (dn->inode_folio_locked) - folio_lock(dn->inode_folio); + else if (dn->inode_entry_locked) + f2fs_lock_cache(dn->inode_entry); return 0; truncate_out: if (f2fs_data_blkaddr(&tdn) == blkaddr) f2fs_truncate_data_blocks_range(&tdn, 1); - if (dn->inode->i_ino == nid && !dn->inode_folio_locked) - folio_unlock(dn->inode_folio); + if (dn->inode->i_ino == nid && !dn->inode_entry_locked) + f2fs_unlock_cache(dn->inode_entry); return 0; } @@ -645,7 +647,7 @@ static int f2fs_reserve_new_block_retry(struct dnode_of_data *dn) } static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, - struct folio *folio) + struct f2fs_cached_block *entry) { struct dnode_of_data dn; struct node_info ni; @@ -653,19 +655,19 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, int err = 0, recovered = 0; /* step 1: recover xattr */ - if (IS_INODE(folio)) { - err = f2fs_recover_inline_xattr(inode, folio); + if (IS_INODE(sbi, entry)) { + err = f2fs_recover_inline_xattr(inode, entry); if (err) goto out; - } else if (f2fs_has_xattr_block(ofs_of_node(folio))) { - err = f2fs_recover_xattr_data(inode, folio); + } else if (f2fs_has_xattr_block(ofs_of_node(sbi, entry))) { + err = f2fs_recover_xattr_data(inode, entry); if (!err) recovered++; goto out; } /* step 2: recover inline data */ - err = f2fs_recover_inline_data(inode, folio); + err = f2fs_recover_inline_data(inode, entry); if (err) { if (err == 1) err = 0; @@ -673,8 +675,8 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, } /* step 3: recover data indices */ - start = f2fs_start_bidx_of_node(ofs_of_node(folio), inode); - end = start + ADDRS_PER_PAGE(folio, inode); + start = f2fs_start_bidx_of_node(ofs_of_node(sbi, entry), inode); + end = start + addrs_per_page(inode, IS_INODE(sbi, entry)); set_new_dnode(&dn, inode, NULL, NULL, 0); retry_dn: @@ -687,18 +689,18 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, goto out; } - f2fs_folio_wait_writeback(dn.node_folio, NODE, true, true); + f2fs_cache_wait_writeback(dn.node_entry); err = f2fs_get_node_info(sbi, dn.nid, &ni, false); if (err) goto err; - f2fs_bug_on(sbi, ni.ino != ino_of_node(folio)); + f2fs_bug_on(sbi, ni.ino != ino_of_node(sbi, entry)); - if (ofs_of_node(dn.node_folio) != ofs_of_node(folio)) { + if (ofs_of_node(sbi, dn.node_entry) != ofs_of_node(sbi, entry)) { f2fs_warn(sbi, "Inconsistent ofs_of_node, ino:%llu, ofs:%u, %u", - inode->i_ino, ofs_of_node(dn.node_folio), - ofs_of_node(folio)); + inode->i_ino, ofs_of_node(sbi, dn.node_entry), + ofs_of_node(sbi, entry)); err = -EFSCORRUPTED; f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); fserror_report_file_metadata(dn.inode, err, GFP_NOFS); @@ -709,7 +711,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, block_t src, dest; src = f2fs_data_blkaddr(&dn); - dest = data_blkaddr(dn.inode, folio, dn.ofs_in_node); + dest = data_blkaddr(dn.inode, entry, dn.ofs_in_node); if (__is_valid_data_blkaddr(src) && !f2fs_is_valid_blkaddr(sbi, src, META_POR)) { @@ -734,9 +736,9 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, } if (!file_keep_isize(inode) && - (i_size_read(inode) <= ((loff_t)index << PAGE_SHIFT))) + (i_size_read(inode) <= F2FS_BLK_TO_BYTES(sbi, index))) f2fs_i_size_write(inode, - (loff_t)(index + 1) << PAGE_SHIFT); + F2FS_BLK_TO_BYTES(sbi, index + 1)); /* * dest is reserved block, invalidate src block @@ -784,16 +786,16 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, } } - copy_node_footer(dn.node_folio, folio); - fill_node_footer(dn.node_folio, dn.nid, ni.ino, - ofs_of_node(folio), false); - folio_mark_dirty(dn.node_folio); + copy_node_footer(sbi, dn.node_entry, entry); + fill_node_footer(sbi, dn.node_entry, dn.nid, ni.ino, + ofs_of_node(sbi, entry), false); + f2fs_mark_cache_dirty(dn.node_entry); err: f2fs_put_dnode(&dn); out: f2fs_notice(sbi, "recover_data: ino = %llx, nid = %x (i_size: %s), " "range (%u, %u), recovered = %d, err = %d", - inode->i_ino, nid_of_node(folio), + inode->i_ino, nid_of_node(sbi, entry), file_keep_isize(inode) ? "keep" : "recover", start, end, recovered, err); return err; @@ -820,26 +822,26 @@ static int recover_data(struct f2fs_sb_info *sbi, struct list_head *inode_list, blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); while (1) { - struct fsync_inode_entry *entry; - struct folio *folio; + struct fsync_inode_entry *fsync_entry; + struct f2fs_cached_block *entry; if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR)) break; - folio = f2fs_get_tmp_folio(sbi, blkaddr); - if (IS_ERR(folio)) { - err = PTR_ERR(folio); + entry = f2fs_get_tmp_cache(sbi, blkaddr); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); break; } - if (!is_recoverable_dnode(folio)) { - f2fs_folio_put(folio, true); + if (!is_recoverable_dnode(sbi, entry)) { + f2fs_put_cache(entry, true); break; } recoverable_dnode++; - entry = get_fsync_inode(inode_list, ino_of_node(folio)); - if (!entry) + fsync_entry = get_fsync_inode(inode_list, ino_of_node(sbi, entry)); + if (!fsync_entry) goto next; fsynced_dnode++; /* @@ -847,40 +849,40 @@ static int recover_data(struct f2fs_sb_info *sbi, struct list_head *inode_list, * In this case, we can lose the latest inode(x). * So, call recover_inode for the inode update. */ - if (IS_INODE(folio)) { - err = recover_inode(entry->inode, folio); + if (IS_INODE(sbi, entry)) { + err = recover_inode(fsync_entry->inode, entry); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); break; } recovered_inode++; } - if (entry->last_dentry == blkaddr) { - err = recover_dentry(entry->inode, folio, dir_list); + if (fsync_entry->last_dentry == blkaddr) { + err = recover_dentry(fsync_entry->inode, entry, dir_list); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); break; } recovered_dentry++; } - err = do_recover_data(sbi, entry->inode, folio); + err = do_recover_data(sbi, fsync_entry->inode, entry); if (err) { - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); break; } recovered_dnode++; - if (entry->blkaddr == blkaddr) - list_move_tail(&entry->list, tmp_inode_list); + if (fsync_entry->blkaddr == blkaddr) + list_move_tail(&fsync_entry->list, tmp_inode_list); next: ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, blkaddr, - next_blkaddr_of_node(folio)); + next_blkaddr_of_node(sbi, entry)); /* check next segment */ - blkaddr = next_blkaddr_of_node(folio); - f2fs_folio_put(folio, true); + blkaddr = next_blkaddr_of_node(sbi, entry); + f2fs_put_cache(entry, true); - f2fs_ra_meta_pages_cond(sbi, blkaddr, ra_blocks); + f2fs_ra_meta_caches_cond(sbi, blkaddr, ra_blocks); total_dnode++; } if (!err) @@ -937,12 +939,11 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) destroy_fsync_dnodes(&tmp_inode_list, err); /* truncate meta pages to be used by the recovery */ - truncate_inode_pages_range(META_MAPPING(sbi), - (loff_t)MAIN_BLKADDR(sbi) << PAGE_SHIFT, -1); - + f2fs_truncate_meta_caches(sbi, MAIN_BLKADDR(sbi), + MAX_BLKADDR(sbi) - MAIN_BLKADDR(sbi)); if (err) { - truncate_inode_pages_final(NODE_MAPPING(sbi)); - truncate_inode_pages_final(META_MAPPING(sbi)); + f2fs_truncate_node_caches(sbi, 0, ULONG_MAX); + f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX); } /* diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 8c156e1fd37d..43f400715766 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -194,7 +194,7 @@ void f2fs_abort_atomic_write(struct inode *inode, bool clean) return; if (clean) - truncate_inode_pages_final(inode->i_mapping); + truncate_inode_pages(inode->i_mapping, 0); release_atomic_write_cnt(inode); clear_inode_flag(inode, FI_ATOMIC_COMMITTED); @@ -335,8 +335,8 @@ static int __f2fs_commit_atomic_write(struct inode *inode) goto next; } - blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, cow_inode), - len); + blen = min((pgoff_t)addrs_per_page(cow_inode, + IS_INODE(F2FS_I_SB(cow_inode), dn.node_entry)), len); index = off; for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) { blkaddr = f2fs_data_blkaddr(&dn); @@ -479,11 +479,11 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi) { int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2; - unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS); - unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA); - unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES); - unsigned int meta = get_pages(sbi, F2FS_DIRTY_META); - unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA); + unsigned int dents = get_nr_caches(sbi, F2FS_DIRTY_DENTS); + unsigned int qdata = get_nr_caches(sbi, F2FS_DIRTY_QDATA); + unsigned int nodes = get_nr_caches(sbi, F2FS_DIRTY_NODES); + unsigned int meta = get_nr_caches(sbi, F2FS_DIRTY_META); + unsigned int imeta = get_nr_caches(sbi, F2FS_DIRTY_IMETA); unsigned int threshold = SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD)); unsigned int global_threshold = threshold * 3 / 2; @@ -510,12 +510,20 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) f2fs_shrink_age_extent_tree(sbi, AGE_EXTENT_CACHE_SHRINK_NUMBER); + /* try to shrink meta cache when there is no enough memory */ + if (!f2fs_available_free_memory(sbi, META_BLOCK)) + f2fs_shrink_meta_cache(sbi, METADATA_CACHE_SHRINK_NUMBER); + + /* try to shrink node cache when there is no enough memory */ + if (!f2fs_available_free_memory(sbi, NODE_BLOCK)) + f2fs_shrink_node_cache(sbi, METADATA_CACHE_SHRINK_NUMBER); + /* check the # of cached NAT entries */ if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) - f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); + f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK(sbi)); if (!f2fs_available_free_memory(sbi, FREE_NIDS)) - f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); + f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS(sbi)); else f2fs_build_free_nids(sbi, false, false); @@ -1310,13 +1318,14 @@ static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi, /* sanity check on discard range */ __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len); - bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start); + bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, dc->di.start); bio->bi_private = dc; bio->bi_end_io = f2fs_submit_discard_endio; submit_bio(bio); atomic_inc(&dcc->issued_discard); - f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, + dc->di.len * F2FS_BLKSIZE(sbi)); } #endif @@ -1327,7 +1336,7 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi, { struct block_device *bdev = dc->bdev; unsigned int max_discard_blocks = - SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev)); struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? &(dcc->fstrim_list) : &(dcc->wait_list); @@ -1389,8 +1398,8 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi, dc->di.len += len; - __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(start), - SECTOR_FROM_BLOCK(len), GFP_NOFS, &bio); + __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(sbi, start), + SECTOR_FROM_BLOCK(sbi, len), GFP_NOFS, &bio); f2fs_bug_on(sbi, !bio); /* @@ -1419,7 +1428,8 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi, atomic_inc(&dcc->issued_discard); - f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, + len * F2FS_BLKSIZE(sbi)); lstart += len; start += len; @@ -1518,7 +1528,7 @@ static void __update_discard_tree_range(struct f2fs_sb_info *sbi, struct discard_info di = {0}; struct rb_node **insert_p = NULL, *insert_parent = NULL; unsigned int max_discard_blocks = - SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev)); block_t end = lstart + len; dc = __lookup_discard_cmd_ret(&dcc->root, lstart, @@ -1871,6 +1881,60 @@ static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, return discard_blks; } +void f2fs_drop_discard_cmd_range(struct f2fs_sb_info *sbi, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + struct discard_cmd *dc, *wait_dc; + u64 cur = start; + u64 end = (u64)start + len; + int count; + + if (!f2fs_realtime_discard_enable(sbi)) + return; + +next: + count = 0; + wait_dc = NULL; + + mutex_lock(&dcc->cmd_lock); + while (cur < end) { + dc = __lookup_discard_cmd_ret(&dcc->root, cur, + &prev_dc, &next_dc, &insert_p, &insert_parent); + if (!dc) + dc = next_dc; + + if (!dc || (u64)dc->di.lstart >= end) + break; + + if (dc->state == D_PREP) { + cur = (u64)dc->di.lstart + dc->di.len; + __remove_discard_cmd(sbi, dc); + if (++count >= MAX_DISCARD_DROP_COUNT) + break; + continue; + } + + dc->ref++; + cur = (u64)dc->di.lstart + dc->di.len; + wait_dc = dc; + break; + } + mutex_unlock(&dcc->cmd_lock); + + if (wait_dc) { + __wait_one_discard_bio(sbi, wait_dc); + goto next; + } + + if (count >= MAX_DISCARD_DROP_COUNT) { + cond_resched(); + goto next; + } +} + /* This should be covered by global mutex, &sit_i->sentry_lock */ static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) { @@ -2041,8 +2105,8 @@ static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, /* For sequential zones, reset the zone write pointer */ if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { - sector = SECTOR_FROM_BLOCK(blkstart); - nr_sects = SECTOR_FROM_BLOCK(blklen); + sector = SECTOR_FROM_BLOCK(sbi, blkstart); + nr_sects = SECTOR_FROM_BLOCK(sbi, blklen); div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder); if (remainder || nr_sects != bdev_zone_sectors(bdev)) { @@ -2749,12 +2813,12 @@ static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int typ } /* - * Calculate the number of current summary pages for writing + * Calculate the number of current summary blocks for writing */ -int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) +int f2fs_nblocks_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) { int valid_sum_count = 0; - int i, sum_in_page; + int i, sum_in_block; for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { if (sbi->ckpt->alloc_type[i] != SSR && for_ra) @@ -2764,73 +2828,66 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); } - sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - + sum_in_block = (sbi->blocksize - 2 * sbi->sum_journal_size - SUM_FOOTER_SIZE) / SUMMARY_SIZE; - if (valid_sum_count <= sum_in_page) + if (valid_sum_count <= sum_in_block) return 1; - else if ((valid_sum_count - sum_in_page) <= + else if ((valid_sum_count - sum_in_block) <= (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) return 2; return 3; } -/* - * Caller should put this summary folio - */ -struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno) + +struct f2fs_cached_block *f2fs_get_sum_cache(struct f2fs_sb_info *sbi, + unsigned int segno) { if (unlikely(f2fs_cp_error(sbi))) return ERR_PTR(-EIO); - return f2fs_get_meta_folio_retry(sbi, GET_SUM_BLOCK(sbi, segno)); + return f2fs_get_meta_cache_retry(sbi, GET_SUM_BLOCK(sbi, segno)); } -void f2fs_update_meta_page(struct f2fs_sb_info *sbi, +void f2fs_update_meta_block(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) { - struct folio *folio; + struct f2fs_cached_block *entry; - if (!f2fs_sb_has_packed_ssa(sbi)) - folio = f2fs_grab_meta_folio(sbi, blk_addr); - else - folio = f2fs_get_meta_folio_retry(sbi, blk_addr); + entry = f2fs_grab_meta_cache(sbi, blk_addr); - if (IS_ERR(folio)) - return; - - memcpy(folio_address(folio), src, PAGE_SIZE); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + memcpy(cache_address(entry), src, F2FS_BLKSIZE(sbi)); + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); } -static void write_sum_page(struct f2fs_sb_info *sbi, +static void write_sum_block(struct f2fs_sb_info *sbi, struct f2fs_summary_block *sum_blk, unsigned int segno) { - struct folio *folio; + struct f2fs_cached_block *entry; if (!f2fs_sb_has_packed_ssa(sbi)) - return f2fs_update_meta_page(sbi, (void *)sum_blk, + return f2fs_update_meta_block(sbi, (void *)sum_blk, GET_SUM_BLOCK(sbi, segno)); - folio = f2fs_get_sum_folio(sbi, segno); - if (IS_ERR(folio)) + entry = f2fs_get_sum_cache(sbi, segno); + if (IS_ERR(entry)) return; - memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + memcpy(SUM_BLK_ENTRY_ADDR(sbi, entry, segno), sum_blk, sbi->sum_blocksize); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); } -static void write_current_sum_page(struct f2fs_sb_info *sbi, +static void write_current_sum_block(struct f2fs_sb_info *sbi, int type, block_t blk_addr) { struct curseg_info *curseg = CURSEG_I(sbi, type); - struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); + struct f2fs_cached_block *entry = f2fs_grab_meta_cache(sbi, blk_addr); struct f2fs_summary_block *src = curseg->sum_blk; struct f2fs_summary_block *dst; - dst = folio_address(folio); - memset(dst, 0, PAGE_SIZE); + dst = cache_address(entry); + memset(dst, 0, sbi->blocksize); mutex_lock(&curseg->curseg_mutex); @@ -2843,8 +2900,8 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, mutex_unlock(&curseg->curseg_mutex); - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); } static int is_next_segment_free(struct f2fs_sb_info *sbi, @@ -2926,7 +2983,7 @@ static int get_new_segment(struct f2fs_sb_info *sbi, alloc_hint = MAIN_SECS(sbi); if (pinning) { - max_secno = sbi->pinned_area_max_secno; + max_secno = min(sbi->pinned_area_max_secno, MAIN_SECS(sbi)); hint = 0; } else if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && hint < alloc_hint) { @@ -3104,7 +3161,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) int ret; if (curseg->inited) - write_sum_page(sbi, curseg->sum_blk, segno); + write_sum_block(sbi, curseg->sum_blk, segno); segno = __get_next_segno(sbi, type); ret = get_new_segment(sbi, &segno, new_sec, pinning); @@ -3160,10 +3217,10 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) struct curseg_info *curseg = CURSEG_I(sbi, type); unsigned int new_segno = curseg->next_segno; struct f2fs_summary_block *sum_node; - struct folio *sum_folio; + struct f2fs_cached_block *entry = NULL; if (curseg->inited) - write_sum_page(sbi, curseg->sum_blk, curseg->segno); + write_sum_block(sbi, curseg->sum_blk, curseg->segno); __set_test_and_inuse(sbi, new_segno); @@ -3176,15 +3233,16 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) curseg->alloc_type = SSR; curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); - sum_folio = f2fs_get_sum_folio(sbi, new_segno); - if (IS_ERR(sum_folio)) { - /* GC won't be able to use stale summary pages by cp_error */ + entry = f2fs_get_sum_cache(sbi, new_segno); + if (IS_ERR(entry)) { + /* GC won't be able to use stale summary blocks by cp_error */ memset(curseg->sum_blk, 0, sbi->sum_entry_size); - return PTR_ERR(sum_folio); + reset_curseg_fields(curseg); + return PTR_ERR(entry); } - sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + sum_node = SUM_BLK_ENTRY_ADDR(sbi, entry, new_segno); memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); - f2fs_folio_put(sum_folio, true); + f2fs_put_cache(entry, true); return 0; } @@ -3271,7 +3329,7 @@ static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) goto out; if (get_valid_blocks(sbi, curseg->segno, false)) { - write_sum_page(sbi, curseg->sum_blk, curseg->segno); + write_sum_block(sbi, curseg->sum_blk, curseg->segno); } else { mutex_lock(&DIRTY_I(sbi)->seglist_lock); __set_test_and_free(sbi, curseg->segno, true); @@ -3592,8 +3650,8 @@ static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) { - __u64 start = F2FS_BYTES_TO_BLK(range->start); - __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; + __u64 start = F2FS_BYTES_TO_BLK(sbi, range->start); + __u64 end = start + F2FS_BYTES_TO_BLK(sbi, range->len) - 1; unsigned int start_segno, end_segno; block_t start_block, end_block; struct cp_control cpc; @@ -3624,7 +3682,8 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) } cpc.reason = CP_DISCARD; - cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); + cpc.trim_minlen = max_t(__u64, 1, + F2FS_BYTES_TO_BLK(sbi, range->minlen)); cpc.trim_start = start_segno; cpc.trim_end = end_segno; @@ -3658,7 +3717,7 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) start_block, end_block); out: if (!err) - range->len = F2FS_BLK_TO_BYTES(trimmed); + range->len = F2FS_BLK_TO_BYTES(sbi, trimmed); return err; } @@ -3770,7 +3829,8 @@ static int __get_segment_type_4(struct f2fs_io_info *fio) else return CURSEG_COLD_DATA; } else { - if (IS_DNODE(fio->folio) && is_cold_node(fio->folio)) + f2fs_bug_on(fio->sbi, !fio->is_cache); + if (IS_DNODE(fio->sbi, fio->cache_entry) && is_cold_node(fio->sbi, fio->cache_entry)) return CURSEG_WARM_NODE; else return CURSEG_COLD_NODE; @@ -3828,9 +3888,9 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), inode->i_write_hint); } else { - if (IS_DNODE(fio->folio)) - return is_cold_node(fio->folio) ? CURSEG_WARM_NODE : - CURSEG_HOT_NODE; + f2fs_bug_on(fio->sbi, !fio->is_cache); + if (IS_DNODE(fio->sbi, fio->cache_entry)) + return is_cold_node(fio->sbi, fio->cache_entry) ? CURSEG_WARM_NODE : CURSEG_HOT_NODE; return CURSEG_COLD_NODE; } } @@ -3897,7 +3957,7 @@ static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, get_random_u32_inclusive(1, sbi->max_fragment_hole); } -int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, +int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, block_t old_blkaddr, block_t *new_blkaddr, struct f2fs_summary *sum, int type, struct f2fs_io_info *fio) @@ -3966,7 +4026,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, if (segment_full) { if (type == CURSEG_COLD_DATA_PINNED && !((curseg->segno + 1) % sbi->segs_per_sec)) { - write_sum_page(sbi, curseg->sum_blk, curseg->segno); + write_sum_block(sbi, curseg->sum_blk, curseg->segno); reset_curseg_fields(curseg); goto skip_new_segment; } @@ -4005,10 +4065,10 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, up_write(&sit_i->sentry_lock); - if (folio && IS_NODESEG(curseg->seg_type)) { - fill_node_footer_blkaddr(folio, NEXT_FREE_BLKADDR(sbi, curseg)); - - f2fs_inode_chksum_set(sbi, folio); + if (fio && fio->is_cache && IS_NODESEG(curseg->seg_type)) { + fill_node_footer_blkaddr(sbi, fio->cache_entry, + NEXT_FREE_BLKADDR(sbi, curseg)); + f2fs_inode_chksum_set(sbi, fio->cache_entry); } if (fio) { @@ -4084,7 +4144,7 @@ static int log_type_to_seg_type(enum log_type type) return seg_type; } -static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) +static void do_write_block(struct f2fs_summary *sum, struct f2fs_io_info *fio) { struct folio *folio = fio->folio; enum log_type type = __get_segment_type(fio); @@ -4096,16 +4156,24 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) if (keep_order) f2fs_down_read(&fio->sbi->io_order_lock); - err = f2fs_allocate_data_block(fio->sbi, folio, fio->old_blkaddr, + err = f2fs_allocate_data_block(fio->sbi, fio->old_blkaddr, &fio->new_blkaddr, sum, type, fio); if (unlikely(err)) { - f2fs_err_ratelimited(fio->sbi, - "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", - __func__, fio->ino, folio->index, type, - fio->old_blkaddr, fio->new_blkaddr, err); - folio_end_writeback(folio); - if (f2fs_in_warm_node_list(folio)) - f2fs_del_fsync_node_entry(fio->sbi, folio); + if (fio->is_cache) { + f2fs_err_ratelimited(fio->sbi, + "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", + __func__, fio->ino, fio->cache_entry->index, type, + fio->old_blkaddr, fio->new_blkaddr, err); + if (f2fs_in_warm_node_list(fio->sbi, fio->cache_entry)) + f2fs_del_fsync_node_entry(fio->sbi, fio->cache_entry); + f2fs_end_cache_writeback(fio->cache_entry); + } else { + f2fs_err_ratelimited(fio->sbi, + "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", + __func__, fio->ino, folio->index, type, + fio->old_blkaddr, fio->new_blkaddr, err); + folio_end_writeback(folio); + } f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi, CP_ERROR_FLAG)); goto out; @@ -4118,7 +4186,10 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1); /* writeout dirty page into bdev */ - f2fs_submit_page_write(fio); + if (fio->is_cache) + f2fs_submit_cache_write(fio); + else + f2fs_submit_page_write(fio); f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); out: @@ -4126,8 +4197,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) f2fs_up_read(&fio->sbi->io_order_lock); } -void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, - enum iostat_type io_type) +void f2fs_do_write_meta_cache(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, + enum iostat_type io_type) { struct f2fs_io_info fio = { .sbi = sbi, @@ -4135,31 +4207,36 @@ void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, .temp = HOT, .op = REQ_OP_WRITE, .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, - .old_blkaddr = folio->index, - .new_blkaddr = folio->index, - .folio = folio, + .old_blkaddr = entry->index, + .new_blkaddr = entry->index, .encrypted_page = NULL, .in_list = 0, + .cache_entry = entry, + .is_cache = 1, }; - if (unlikely(folio->index >= MAIN_BLKADDR(sbi))) + if (unlikely(entry->index >= MAIN_BLKADDR(sbi))) fio.op_flags &= ~REQ_META; - folio_start_writeback(folio); - f2fs_submit_page_write(&fio); + f2fs_start_cache_writeback(entry); + f2fs_submit_cache_write(&fio); - stat_inc_meta_count(sbi, folio->index); - f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE); + stat_inc_meta_count(sbi, entry->index); + f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE(sbi)); } -void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) +void f2fs_do_write_node_cache(unsigned int nid, struct f2fs_io_info *fio) { struct f2fs_summary sum; - set_summary(&sum, nid, 0, 0); - do_write_page(&sum, fio); + if (fio->is_cache) + trace_f2fs_write_cache(fio->cache_entry, NODE); - f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE); + set_summary(&sum, nid, 0, 0); + do_write_block(&sum, fio); + + f2fs_update_iostat(fio->sbi, NULL, fio->io_type, + F2FS_BLKSIZE(fio->sbi)); } void f2fs_outplace_write_data(struct dnode_of_data *dn, @@ -4172,10 +4249,11 @@ void f2fs_outplace_write_data(struct dnode_of_data *dn, if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) f2fs_update_age_extent_cache(dn); set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); - do_write_page(&sum, fio); + do_write_block(&sum, fio); f2fs_update_data_blkaddr(dn, fio->new_blkaddr); - f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, dn->inode, fio->io_type, + F2FS_BLKSIZE(sbi)); } int f2fs_inplace_write_data(struct f2fs_io_info *fio) @@ -4205,7 +4283,7 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) } if (fio->meta_gc) - f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1); + f2fs_truncate_meta_caches(sbi, fio->new_blkaddr, 1); stat_inc_inplace_blocks(fio->sbi); @@ -4214,10 +4292,11 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) else err = f2fs_submit_page_bio(fio); if (!err) { + fio->submitted = 1; f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); f2fs_update_iostat(fio->sbi, fio_inode(fio), - fio->io_type, F2FS_BLKSIZE); + fio->io_type, F2FS_BLKSIZE(fio->sbi)); } return err; @@ -4349,14 +4428,13 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, f2fs_update_data_blkaddr(dn, new_addr); } -void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, - bool ordered, bool locked) +void f2fs_folio_wait_writeback(struct folio *folio, bool ordered, bool locked) { if (folio_test_writeback(folio)) { struct f2fs_sb_info *sbi = F2FS_F_SB(folio); /* submit cached LFS IO */ - f2fs_submit_merged_write_folio(sbi, folio, type); + f2fs_submit_merged_write_folio(sbi, folio); /* submit cached IPU IO */ f2fs_submit_merged_ipu_write(sbi, NULL, folio); if (ordered) { @@ -4371,7 +4449,7 @@ void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct folio *cfolio; + struct f2fs_cached_block *entry; if (!f2fs_meta_inode_gc_required(inode)) return; @@ -4379,11 +4457,12 @@ void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) if (!__is_valid_data_blkaddr(blkaddr)) return; - cfolio = filemap_lock_folio(META_MAPPING(sbi), blkaddr); - if (!IS_ERR(cfolio)) { - f2fs_folio_wait_writeback(cfolio, DATA, true, true); - f2fs_folio_put(cfolio, true); - } + entry = f2fs_find_cache(META_CACHE(sbi), blkaddr, 0); + if (IS_ERR(entry)) + return; + f2fs_lock_cache(entry); + f2fs_cache_wait_writeback_cond(entry, DATA); + f2fs_put_cache(entry, true); } void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, @@ -4398,7 +4477,7 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, for (i = 0; i < len; i++) f2fs_wait_on_block_writeback(inode, blkaddr + i); - f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); + f2fs_truncate_meta_caches(sbi, blkaddr, len); } static int read_compacted_summaries(struct f2fs_sb_info *sbi) @@ -4406,16 +4485,16 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); struct curseg_info *seg_i; unsigned char *kaddr; - struct folio *folio; + struct f2fs_cached_block *entry; block_t start; int i, j, offset; start = start_sum_block(sbi); - folio = f2fs_get_meta_folio(sbi, start++); - if (IS_ERR(folio)) - return PTR_ERR(folio); - kaddr = folio_address(folio); + entry = f2fs_get_meta_cache(sbi, start++); + if (IS_ERR(entry)) + return PTR_ERR(entry); + kaddr = cache_address(entry); /* Step 1: restore nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); @@ -4452,16 +4531,16 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) SUM_FOOTER_SIZE) continue; - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); - folio = f2fs_get_meta_folio(sbi, start++); - if (IS_ERR(folio)) - return PTR_ERR(folio); - kaddr = folio_address(folio); + entry = f2fs_get_meta_cache(sbi, start++); + if (IS_ERR(entry)) + return PTR_ERR(entry); + kaddr = cache_address(entry); offset = 0; } } - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); return 0; } @@ -4470,7 +4549,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); struct f2fs_summary_block *sum; struct curseg_info *curseg; - struct folio *new; + struct f2fs_cached_block *entry; unsigned short blk_off; unsigned int segno = 0; block_t blk_addr = 0; @@ -4497,10 +4576,10 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) blk_addr = GET_SUM_BLOCK(sbi, segno); } - new = f2fs_get_meta_folio(sbi, blk_addr); - if (IS_ERR(new)) - return PTR_ERR(new); - sum = folio_address(new); + entry = f2fs_get_meta_cache(sbi, blk_addr); + if (IS_ERR(entry)) + return PTR_ERR(entry); + sum = cache_address(entry); if (IS_NODESEG(type)) { if (__exist_node_summaries(sbi)) { @@ -4537,7 +4616,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) curseg->next_blkoff = blk_off; mutex_unlock(&curseg->curseg_mutex); out: - f2fs_folio_put(new, true); + f2fs_put_cache(entry, true); return err; } @@ -4549,11 +4628,11 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) int err; if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { - int npages = f2fs_npages_for_summary_flush(sbi, true); + int nblocks = f2fs_nblocks_for_summary_flush(sbi, true); - if (npages >= 2) - f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, - META_CP, true); + if (nblocks >= 2) + f2fs_ra_meta_caches(sbi, start_sum_block(sbi), + nblocks, META_CP, true); /* restore for compacted data summary */ err = read_compacted_summaries(sbi); @@ -4563,7 +4642,7 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) } if (__exist_node_summaries(sbi)) - f2fs_ra_meta_pages(sbi, + f2fs_ra_meta_caches(sbi, sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), NR_CURSEG_PERSIST_TYPE - type, META_CP, true); @@ -4586,16 +4665,16 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) { - struct folio *folio; + struct f2fs_cached_block *entry = NULL; unsigned char *kaddr; struct f2fs_summary *summary; struct curseg_info *seg_i; int written_size = 0; int i, j; - folio = f2fs_grab_meta_folio(sbi, blkaddr++); - kaddr = folio_address(folio); - memset(kaddr, 0, PAGE_SIZE); + entry = f2fs_grab_meta_cache(sbi, blkaddr++); + kaddr = cache_address(entry); + memset(kaddr, 0, sbi->blocksize); /* Step 1: write nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); @@ -4611,10 +4690,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { seg_i = CURSEG_I(sbi, i); for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { - if (!folio) { - folio = f2fs_grab_meta_folio(sbi, blkaddr++); - kaddr = folio_address(folio); - memset(kaddr, 0, PAGE_SIZE); + if (!entry) { + entry = f2fs_grab_meta_cache(sbi, blkaddr++); + kaddr = cache_address(entry); + memset(kaddr, 0, sbi->blocksize); written_size = 0; } summary = (struct f2fs_summary *)(kaddr + written_size); @@ -4625,14 +4704,14 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) SUM_FOOTER_SIZE) continue; - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); - folio = NULL; + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); + entry = NULL; } } - if (folio) { - folio_mark_dirty(folio); - f2fs_folio_put(folio, true); + if (entry) { + f2fs_mark_cache_dirty(entry); + f2fs_put_cache(entry, true); } } @@ -4647,7 +4726,7 @@ static void write_normal_summaries(struct f2fs_sb_info *sbi, end = type + NR_CURSEG_NODE_TYPE; for (i = type; i < end; i++) - write_current_sum_page(sbi, i, blkaddr + (i - type)); + write_current_sum_block(sbi, i, blkaddr + (i - type)); } void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) @@ -4686,29 +4765,29 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, return -1; } -static struct folio *get_current_sit_folio(struct f2fs_sb_info *sbi, +static struct f2fs_cached_block *get_current_sit_cache(struct f2fs_sb_info *sbi, unsigned int segno) { - return f2fs_get_meta_folio(sbi, current_sit_addr(sbi, segno)); + return f2fs_get_meta_cache(sbi, current_sit_addr(sbi, segno)); } -static struct folio *get_next_sit_folio(struct f2fs_sb_info *sbi, +static struct f2fs_cached_block *get_next_sit_cache(struct f2fs_sb_info *sbi, unsigned int start) { struct sit_info *sit_i = SIT_I(sbi); - struct folio *folio; + struct f2fs_cached_block *entry; pgoff_t src_off, dst_off; src_off = current_sit_addr(sbi, start); dst_off = next_sit_addr(sbi, src_off); - folio = f2fs_grab_meta_folio(sbi, dst_off); - seg_info_to_sit_folio(sbi, folio, start); + entry = f2fs_grab_meta_cache(sbi, dst_off); + seg_info_to_sit_block(sbi, entry, start); - folio_mark_dirty(folio); - set_to_next_sit(sit_i, start); + f2fs_mark_cache_dirty(entry); + set_to_next_sit(sbi, sit_i, start); - return folio; + return entry; } static struct sit_entry_set *grab_sit_entry_set(void) @@ -4745,10 +4824,11 @@ static void adjust_sit_entry_set(struct sit_entry_set *ses, list_move_tail(&ses->set_list, head); } -static void add_sit_entry(unsigned int segno, struct list_head *head) +static void add_sit_entry(struct f2fs_sb_info *sbi, unsigned int segno, + struct list_head *head) { struct sit_entry_set *ses; - unsigned int start_segno = START_SEGNO(segno); + unsigned int start_segno = f2fs_start_segno(sbi, segno); list_for_each_entry(ses, head, set_list) { if (ses->start_segno == start_segno) { @@ -4773,7 +4853,7 @@ static void add_sits_in_set(struct f2fs_sb_info *sbi) unsigned int segno; for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) - add_sit_entry(segno, set_list); + add_sit_entry(sbi, segno, set_list); } static void remove_sits_in_journal(struct f2fs_sb_info *sbi) @@ -4791,7 +4871,7 @@ static void remove_sits_in_journal(struct f2fs_sb_info *sbi) dirtied = __mark_sit_entry_dirty(sbi, segno); if (!dirtied) - add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); + add_sit_entry(sbi, segno, &SM_I(sbi)->sit_entry_set); } update_sits_in_cursum(journal, -i); up_write(&curseg->journal_rwsem); @@ -4814,7 +4894,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) down_write(&sit_i->sentry_lock); - if (!sit_i->dirty_sentries) + if (!sit_i->dirty_sentries && to_journal) goto out; /* @@ -4838,10 +4918,11 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * #2, flush sit entries to sit page. */ list_for_each_entry_safe(ses, tmp, head, set_list) { - struct folio *folio = NULL; + struct f2fs_cached_block *entry = NULL; struct f2fs_sit_block *raw_sit = NULL; + unsigned int start_segno = ses->start_segno; - unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, + unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK(sbi), (unsigned long)MAIN_SEGS(sbi)); unsigned int segno = start_segno; @@ -4853,8 +4934,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) if (to_journal) { down_write(&curseg->journal_rwsem); } else { - folio = get_next_sit_folio(sbi, start_segno); - raw_sit = folio_address(folio); + entry = get_next_sit_cache(sbi, start_segno); + raw_sit = cache_address(entry); } /* flush dirty sit entries in region of current sit set */ @@ -4899,7 +4980,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) if (to_journal) up_write(&curseg->journal_rwsem); else - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); f2fs_bug_on(sbi, ses->entry_cnt); release_sit_entry_set(ses); @@ -5006,7 +5087,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi) sit_i->written_valid_blocks = 0; sit_i->bitmap_size = sit_bitmap_size; sit_i->dirty_sentries = 0; - sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; + sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK(sbi); sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); sit_i->mounted_time = ktime_get_boottime_seconds(); init_rwsem(&sit_i->sentry_lock); @@ -5090,7 +5171,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) block_t sit_valid_blocks[2] = {0, 0}; do { - readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, + readed = f2fs_ra_meta_caches(sbi, start_blk, BIO_MAX_VECS, META_SIT, true); start = start_blk * sit_i->sents_per_block; @@ -5098,15 +5179,15 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) for (; start < end && start < MAIN_SEGS(sbi); start++) { struct f2fs_sit_block *sit_blk; - struct folio *folio; + struct f2fs_cached_block *entry; se = &sit_i->sentries[start]; - folio = get_current_sit_folio(sbi, start); - if (IS_ERR(folio)) - return PTR_ERR(folio); - sit_blk = folio_address(folio); + entry = get_current_sit_cache(sbi, start); + if (IS_ERR(entry)) + return PTR_ERR(entry); + sit_blk = cache_address(entry); sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; - f2fs_folio_put(folio, true); + f2fs_put_cache(entry, true); err = check_block_count(sbi, start, &sit); if (err) diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 5949aa5200ac..918bc4452ad0 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -91,30 +91,46 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) +static inline void f2fs_adjust_pinned_area_boundary(struct f2fs_sb_info *sbi) +{ + sbi->pinned_area_max_secno = MAIN_SECS(sbi); + if (f2fs_sb_has_blkzoned(sbi) && sbi->first_seq_zone_segno != NULL_SEGNO) + sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, + GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno)); + if (F2FS_OPTION(sbi).resizable_tail_secno) + sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, + MAIN_SECS(sbi) - F2FS_OPTION(sbi).resizable_tail_secno); +} + #define GET_SUM_BLOCK(sbi, segno) \ (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) #define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) #define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) +#define SUM_BLK_ENTRY_ADDR(sbi, entry, segno) \ + (cache_address(entry) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) #define GET_SUM_TYPE(footer) ((footer)->entry_type) #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) #define SIT_ENTRY_OFFSET(sit_i, segno) \ ((segno) % (sit_i)->sents_per_block) -#define SIT_BLOCK_OFFSET(segno) \ - ((segno) / SIT_ENTRY_PER_BLOCK) -#define START_SEGNO(segno) \ - (SIT_BLOCK_OFFSET(segno) * SIT_ENTRY_PER_BLOCK) +#define SIT_BLOCK_OFFSET(sbi, segno) \ + ((segno) / SIT_ENTRY_PER_BLOCK(sbi)) +static inline unsigned int +f2fs_start_segno(struct f2fs_sb_info *sbi, unsigned int segno) +{ + return SIT_BLOCK_OFFSET(sbi, segno) * SIT_ENTRY_PER_BLOCK(sbi); +} #define SIT_BLK_CNT(sbi) \ - DIV_ROUND_UP(MAIN_SEGS(sbi), SIT_ENTRY_PER_BLOCK) + DIV_ROUND_UP(MAIN_SEGS(sbi), SIT_ENTRY_PER_BLOCK(sbi)) #define f2fs_bitmap_size(nr) \ (BITS_TO_LONGS(nr) * sizeof(unsigned long)) -#define SECTOR_FROM_BLOCK(blk_addr) \ - (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK) -#define SECTOR_TO_BLOCK(sectors) \ - ((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK) +#define SECTOR_FROM_BLOCK(sbi, blk_addr) \ + (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK(sbi)) +#define SECTOR_TO_BLOCK(sbi, sectors) \ + ((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK(sbi)) /* * In the victim_sel_policy->alloc_mode, there are three block allocation modes. @@ -418,18 +434,18 @@ static inline void __seg_info_to_raw_sit(struct seg_entry *se, rs->mtime = cpu_to_le64(se->mtime); } -static inline void seg_info_to_sit_folio(struct f2fs_sb_info *sbi, - struct folio *folio, unsigned int start) +static inline void seg_info_to_sit_block(struct f2fs_sb_info *sbi, + struct f2fs_cached_block *entry, unsigned int start) { struct f2fs_sit_block *raw_sit; struct seg_entry *se; struct f2fs_sit_entry *rs; - unsigned int end = min(start + SIT_ENTRY_PER_BLOCK, + unsigned int end = min(start + SIT_ENTRY_PER_BLOCK(sbi), (unsigned long)MAIN_SEGS(sbi)); int i; - raw_sit = folio_address(folio); - memset(raw_sit, 0, PAGE_SIZE); + raw_sit = cache_address(entry); + memset(raw_sit, 0, F2FS_BLKSIZE(sbi)); for (i = 0; i < end - start; i++) { rs = &raw_sit->entries[i]; se = get_seg_entry(sbi, start + i); @@ -651,15 +667,15 @@ static inline void get_additional_blocks_required(struct f2fs_sb_info *sbi, */ static inline int __get_secs_required(struct f2fs_sb_info *sbi) { - unsigned int total_node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) + - get_pages(sbi, F2FS_DIRTY_DENTS) + - get_pages(sbi, F2FS_DIRTY_IMETA); - unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int total_node_blocks = get_nr_caches(sbi, F2FS_DIRTY_NODES) + + get_nr_caches(sbi, F2FS_DIRTY_DENTS) + + get_nr_caches(sbi, F2FS_DIRTY_IMETA); + unsigned int total_dent_blocks = get_nr_caches(sbi, F2FS_DIRTY_DENTS); unsigned int total_data_blocks = 0; bool separate_dent = true; if (f2fs_lfs_mode(sbi)) - total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA); + total_data_blocks = get_nr_caches(sbi, F2FS_DIRTY_DATA); /* * When active_logs != 4, dentry blocks and data blocks can be @@ -869,7 +885,7 @@ static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi, unsigned int start) { struct sit_info *sit_i = SIT_I(sbi); - unsigned int offset = SIT_BLOCK_OFFSET(start); + unsigned int offset = SIT_BLOCK_OFFSET(sbi, start); block_t blk_addr = sit_i->sit_base_addr + offset; f2fs_bug_on(sbi, !valid_main_segno(sbi, start)); @@ -894,9 +910,10 @@ static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi, return block_addr + sit_i->sit_base_addr; } -static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start) +static inline void set_to_next_sit(struct f2fs_sb_info *sbi, + struct sit_info *sit_i, unsigned int start) { - unsigned int block_off = SIT_BLOCK_OFFSET(start); + unsigned int block_off = SIT_BLOCK_OFFSET(sbi, start); f2fs_change_bit(block_off, sit_i->sit_bitmap); } @@ -971,13 +988,13 @@ static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno) } /* - * It is very important to gather dirty pages and write at once, so that we can + * It is very important to gather dirty blocks and write at once, so that we can * submit a big bio without interfering other data writes. - * By default, 512 pages for directory data, - * 512 pages (2MB) * 8 for nodes, and - * 256 pages * 8 for meta are set. + * By default, 512 blocks for directory data, + * 512 blocks (2MB) * 8 for nodes, and + * 256 blocks * 8 for meta are set. */ -static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) +static inline int nr_caches_to_skip(struct f2fs_sb_info *sbi, int type) { if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) return 0; @@ -993,23 +1010,24 @@ static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) } /* - * When writing pages, it'd better align nr_to_write for segment size. + * When writing cache asynchronously, align nr_to_write to BIO_MAX_VECS. */ -static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type, - struct writeback_control *wbc) +static inline long adjust_flush_cache_number(struct f2fs_sb_info *sbi, int type) { - long nr_to_write, desired; + long nr_to_write; - if (wbc->sync_mode != WB_SYNC_NONE) + switch (type) { + case META: + nr_to_write = BIO_MAX_VECS; + break; + case NODE: + nr_to_write = BIO_MAX_VECS << 1; + break; + default: + f2fs_bug_on(sbi, 1); return 0; - - nr_to_write = wbc->nr_to_write; - desired = BIO_MAX_VECS; - if (type == NODE) - desired <<= 1; - - wbc->nr_to_write = desired; - return desired - nr_to_write; + } + return nr_to_write; } static inline void wake_up_discard_thread(struct f2fs_sb_info *sbi, bool force) diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c index 4f6bf5926de4..29f488531492 100644 --- a/fs/f2fs/shrinker.c +++ b/fs/f2fs/shrinker.c @@ -23,7 +23,7 @@ static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi) static unsigned long __count_free_nids(struct f2fs_sb_info *sbi) { - long count = NM_I(sbi)->nid_cnt[FREE_NID] - MAX_FREE_NIDS; + long count = NM_I(sbi)->nid_cnt[FREE_NID] - MAX_FREE_NIDS(sbi); return count > 0 ? count : 0; } @@ -37,6 +37,13 @@ static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi, atomic_read(&eti->total_ext_node); } +static unsigned long __count_cache(struct f2fs_sb_info *sbi) +{ + return META_CACHE(sbi)->num_entries + + NODE_CACHE(sbi)->num_entries + + COMPRESS_CACHE(sbi)->num_entries; +} + unsigned long f2fs_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { @@ -68,6 +75,9 @@ unsigned long f2fs_shrink_count(struct shrinker *shrink, /* count free nids cache entries */ count += __count_free_nids(sbi); + /* count generic cache entries */ + count += __count_cache(sbi); + spin_lock(&f2fs_list_lock); p = p->next; mutex_unlock(&sbi->umount_mutex); @@ -120,6 +130,10 @@ unsigned long f2fs_shrink_scan(struct shrinker *shrink, if (freed < nr) freed += f2fs_try_to_free_nids(sbi, nr - freed); + /* shrink generic cache entries */ + if (freed < nr) + freed += f2fs_shrink_cache(sbi, nr - freed); + spin_lock(&f2fs_list_lock); p = p->next; list_move_tail(&sbi->s_list, &f2fs_list); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 1314b6ccced9..1071059f2cc2 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -338,11 +338,11 @@ static const struct fs_parameter_spec f2fs_param_specs[] = { fsparam_flag("usrquota", Opt_usrquota), fsparam_flag("grpquota", Opt_grpquota), fsparam_flag("prjquota", Opt_prjquota), - fsparam_string("usrjquota", Opt_usrjquota), + fsparam_string_empty("usrjquota", Opt_usrjquota), fsparam_flag("usrjquota", Opt_usrjquota), - fsparam_string("grpjquota", Opt_grpjquota), + fsparam_string_empty("grpjquota", Opt_grpjquota), fsparam_flag("grpjquota", Opt_grpjquota), - fsparam_string("prjjquota", Opt_prjjquota), + fsparam_string_empty("prjjquota", Opt_prjjquota), fsparam_flag("prjjquota", Opt_prjjquota), fsparam_flag("nat_bits", Opt_nat_bits), fsparam_enum("jqfmt", Opt_jqfmt, f2fs_param_jqfmt), @@ -420,21 +420,33 @@ struct f2fs_fs_context { static inline void ctx_set_opt(struct f2fs_fs_context *ctx, enum f2fs_mount_opt flag) { - ctx->info.opt |= BIT(flag); - ctx->opt_mask |= BIT(flag); + ctx->info.opt |= BIT_ULL(flag); + ctx->opt_mask |= BIT_ULL(flag); } static inline void ctx_clear_opt(struct f2fs_fs_context *ctx, enum f2fs_mount_opt flag) { - ctx->info.opt &= ~BIT(flag); - ctx->opt_mask |= BIT(flag); + ctx->info.opt &= ~BIT_ULL(flag); + ctx->opt_mask |= BIT_ULL(flag); } static inline bool ctx_test_opt(struct f2fs_fs_context *ctx, enum f2fs_mount_opt flag) { - return ctx->info.opt & BIT(flag); + return ctx->info.opt & BIT_ULL(flag); +} + +static inline void ctx_clear_opt_mask(struct f2fs_fs_context *ctx, + enum f2fs_mount_opt flag) +{ + ctx->opt_mask &= ~BIT_ULL(flag); +} + +static inline bool ctx_test_opt_mask(struct f2fs_fs_context *ctx, + enum f2fs_mount_opt flag) +{ + return ctx->opt_mask & BIT_ULL(flag); } void f2fs_printk(struct f2fs_sb_info *sbi, bool limit_rate, @@ -554,17 +566,6 @@ static inline void adjust_unusable_cap_perc(struct f2fs_sb_info *sbi) F2FS_OPTION(sbi).unusable_cap_perc); } -static inline void adjust_pinned_area_boundary(struct f2fs_sb_info *sbi) -{ - sbi->pinned_area_max_secno = MAIN_SECS(sbi); - if (f2fs_sb_has_blkzoned(sbi) && sbi->first_seq_zone_segno != NULL_SEGNO) - sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, - GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno)); - if (F2FS_OPTION(sbi).resizable_tail_secno) - sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, - MAIN_SECS(sbi) - F2FS_OPTION(sbi).resizable_tail_secno); -} - static void init_once(void *foo) { struct f2fs_inode_info *fi = (struct f2fs_inode_info *) foo; @@ -855,9 +856,11 @@ static int f2fs_parse_param(struct fs_context *fc, struct fs_parameter *param) break; case Opt_inline_xattr_size: if (result.int_32 < MIN_INLINE_XATTR_SIZE || - result.int_32 > MAX_INLINE_XATTR_SIZE) { + result.int_32 > + MAX_INLINE_XATTR_SIZE(F2FS_MAX_BLKSIZE)) { f2fs_err(NULL, "inline xattr size is out of range: %u ~ %u", - (u32)MIN_INLINE_XATTR_SIZE, (u32)MAX_INLINE_XATTR_SIZE); + (u32)MIN_INLINE_XATTR_SIZE, + (u32)MAX_INLINE_XATTR_SIZE(F2FS_MAX_BLKSIZE)); return -EINVAL; } ctx_set_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE); @@ -1441,7 +1444,7 @@ static int f2fs_check_compression(struct fs_context *fc, ctx_test_opt(ctx, F2FS_MOUNT_COMPRESS_CACHE)) f2fs_info(sbi, "Image doesn't support compression"); clear_compression_spec(ctx); - ctx->opt_mask &= ~BIT(F2FS_MOUNT_COMPRESS_CACHE); + ctx_clear_opt_mask(ctx, F2FS_MOUNT_COMPRESS_CACHE); return 0; } if (ctx->spec_mask & F2FS_SPEC_compress_extension) { @@ -1509,43 +1512,43 @@ static int f2fs_check_opt_consistency(struct fs_context *fc, return -EINVAL; if (f2fs_hw_should_discard(sbi) && - (ctx->opt_mask & BIT(F2FS_MOUNT_DISCARD)) && + ctx_test_opt_mask(ctx, F2FS_MOUNT_DISCARD) && !ctx_test_opt(ctx, F2FS_MOUNT_DISCARD)) { f2fs_warn(sbi, "discard is required for zoned block devices"); return -EINVAL; } if (!f2fs_hw_support_discard(sbi) && - (ctx->opt_mask & BIT(F2FS_MOUNT_DISCARD)) && + ctx_test_opt_mask(ctx, F2FS_MOUNT_DISCARD) && ctx_test_opt(ctx, F2FS_MOUNT_DISCARD)) { f2fs_warn(sbi, "device does not support discard"); ctx_clear_opt(ctx, F2FS_MOUNT_DISCARD); - ctx->opt_mask &= ~BIT(F2FS_MOUNT_DISCARD); + ctx_clear_opt_mask(ctx, F2FS_MOUNT_DISCARD); } if (f2fs_sb_has_device_alias(sbi) && - (ctx->opt_mask & BIT(F2FS_MOUNT_READ_EXTENT_CACHE)) && + ctx_test_opt_mask(ctx, F2FS_MOUNT_READ_EXTENT_CACHE) && !ctx_test_opt(ctx, F2FS_MOUNT_READ_EXTENT_CACHE)) { f2fs_err(sbi, "device aliasing requires extent cache"); return -EINVAL; } if (test_opt(sbi, RESERVE_ROOT) && - (ctx->opt_mask & BIT(F2FS_MOUNT_RESERVE_ROOT)) && + ctx_test_opt_mask(ctx, F2FS_MOUNT_RESERVE_ROOT) && ctx_test_opt(ctx, F2FS_MOUNT_RESERVE_ROOT)) { f2fs_info(sbi, "Preserve previous reserve_root=%u", F2FS_OPTION(sbi).root_reserved_blocks); ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_ROOT); - ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_ROOT); + ctx_clear_opt_mask(ctx, F2FS_MOUNT_RESERVE_ROOT); ctx->spec_mask &= ~F2FS_SPEC_reserve_root; } if (test_opt(sbi, RESERVE_NODE) && - (ctx->opt_mask & BIT(F2FS_MOUNT_RESERVE_NODE)) && + ctx_test_opt_mask(ctx, F2FS_MOUNT_RESERVE_NODE) && ctx_test_opt(ctx, F2FS_MOUNT_RESERVE_NODE)) { f2fs_info(sbi, "Preserve previous reserve_node=%u", F2FS_OPTION(sbi).root_reserved_nodes); ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_NODE); - ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_NODE); + ctx_clear_opt_mask(ctx, F2FS_MOUNT_RESERVE_NODE); ctx->spec_mask &= ~F2FS_SPEC_reserve_node; } @@ -1596,6 +1599,8 @@ static int f2fs_check_opt_consistency(struct fs_context *fc, } if (ctx_test_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE)) { + int min_size, max_size; + if (!f2fs_sb_has_extra_attr(sbi) || !f2fs_sb_has_flexible_inline_xattr(sbi)) { f2fs_err(sbi, "extra_attr or flexible_inline_xattr feature is off"); @@ -1605,6 +1610,15 @@ static int f2fs_check_opt_consistency(struct fs_context *fc, f2fs_err(sbi, "inline_xattr_size option should be set with inline_xattr option"); return -EINVAL; } + min_size = MIN_INLINE_XATTR_SIZE; + max_size = MAX_INLINE_XATTR_SIZE(F2FS_BLKSIZE(sbi)); + + if (F2FS_OPTION(sbi).inline_xattr_size < min_size || + F2FS_OPTION(sbi).inline_xattr_size > max_size) { + f2fs_err(sbi, "inline xattr size is out of range: %d ~ %d", + min_size, max_size); + return -EINVAL; + } } if (ctx_test_opt(ctx, F2FS_MOUNT_ATGC) && @@ -1859,21 +1873,8 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) static int f2fs_drop_inode(struct inode *inode) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); int ret; - /* - * during filesystem shutdown, if checkpoint is disabled, - * drop useless meta/node dirty pages. - */ - if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { - if (inode->i_ino == F2FS_NODE_INO(sbi) || - inode->i_ino == F2FS_META_INO(sbi)) { - trace_f2fs_drop_inode(inode, 1); - return 1; - } - } - /* * This is to avoid a deadlock condition like below. * writeback_single_inode(inode) @@ -1894,7 +1895,7 @@ static int f2fs_drop_inode(struct inode *inode) f2fs_i_size_write(inode, 0); f2fs_submit_merged_write_cond(F2FS_I_SB(inode), - inode, NULL, 0, DATA); + inode, NULL); truncate_inode_pages_final(inode->i_mapping); if (F2FS_HAS_BLOCKS(inode)) @@ -1930,7 +1931,7 @@ int f2fs_inode_dirtied(struct inode *inode, bool sync) if (sync && list_empty(&F2FS_I(inode)->gdirty_list)) { list_add_tail(&F2FS_I(inode)->gdirty_list, &sbi->inode_list[DIRTY_META]); - inc_page_count(sbi, F2FS_DIRTY_IMETA); + inc_cache_count(sbi, F2FS_DIRTY_IMETA); } spin_unlock(&sbi->inode_lock[DIRTY_META]); @@ -1953,7 +1954,7 @@ void f2fs_inode_synced(struct inode *inode) } if (!list_empty(&F2FS_I(inode)->gdirty_list)) { list_del_init(&F2FS_I(inode)->gdirty_list); - dec_page_count(sbi, F2FS_DIRTY_IMETA); + dec_cache_count(sbi, F2FS_DIRTY_IMETA); } clear_inode_flag(inode, FI_DIRTY_INODE); clear_inode_flag(inode, FI_AUTO_RECOVER); @@ -1968,12 +1969,6 @@ void f2fs_inode_synced(struct inode *inode) */ static void f2fs_dirty_inode(struct inode *inode, int flags) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - - if (inode->i_ino == F2FS_NODE_INO(sbi) || - inode->i_ino == F2FS_META_INO(sbi)) - return; - if (is_inode_flag_set(inode, FI_AUTO_RECOVER)) clear_inode_flag(inode, FI_AUTO_RECOVER); @@ -2026,6 +2021,7 @@ static void f2fs_put_super(struct super_block *sb) * flush all issued checkpoints and stop checkpoint issue thread. * after then, all checkpoints should be done by each process context. */ + f2fs_stop_cache_wb_thread(sbi); f2fs_stop_ckpt_thread(sbi); /* @@ -2064,30 +2060,27 @@ static void f2fs_put_super(struct super_block *sb) /* our cp_error case, we can wait for any writeback page */ f2fs_flush_merged_writes(sbi); - f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA); - if (err || f2fs_cp_error(sbi)) { - truncate_inode_pages_final(NODE_MAPPING(sbi)); - truncate_inode_pages_final(META_MAPPING(sbi)); + if (err || f2fs_cp_error(sbi) || + unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + f2fs_truncate_node_caches(sbi, 0, ULONG_MAX); + f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX); } f2fs_bug_on(sbi, sbi->fsync_node_num); - f2fs_destroy_compress_inode(sbi); - - iput(sbi->node_inode); - sbi->node_inode = NULL; - - iput(sbi->meta_inode); - sbi->meta_inode = NULL; + f2fs_destroy_cache(COMPRESS_CACHE(sbi)); + f2fs_destroy_cache(NODE_CACHE(sbi)); + f2fs_destroy_cache(META_CACHE(sbi)); /* Should check the page counts after dropping all node/meta pages */ for (i = 0; i < NR_COUNT_TYPE; i++) { - if (!get_pages(sbi, i)) + if (!get_nr_caches(sbi, i)) continue; f2fs_err(sbi, "detect filesystem reference count leak during " "umount, type: %d, count: %lld, err: %d, cp_err: %d", - i, get_pages(sbi, i), err, f2fs_cp_error(sbi)); + i, get_nr_caches(sbi, i), err, f2fs_cp_error(sbi)); f2fs_bug_on(sbi, 1); } @@ -2154,6 +2147,25 @@ int f2fs_sync_fs(struct super_block *sb, int sync) return err; } +static int f2fs_sync_fs_super(struct super_block *sb, int sync) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + bool freeze_sync = sync && + sb->s_writers.frozen == SB_FREEZE_PAGEFAULT; + int err; + + /* freeze_super() holds s_umount for write during this sync pass. */ + if (freeze_sync) + sbi->umount_lock_holder = current; + + err = f2fs_sync_fs(sb, sync); + + if (freeze_sync) + sbi->umount_lock_holder = NULL; + + return err; +} + static int f2fs_freeze(struct super_block *sb) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -2701,9 +2713,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) f2fs_info(sbi, "%s: call sync_filesystem() to persist meta: %lld, node: %lld, data: %lld", __func__, - get_pages(sbi, F2FS_DIRTY_META), - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DATA)); + get_nr_caches(sbi, F2FS_DIRTY_META), + get_nr_caches(sbi, F2FS_DIRTY_NODES), + get_nr_caches(sbi, F2FS_DIRTY_DATA)); ret = sync_filesystem(sbi->sb); if (ret || err) { @@ -2720,9 +2732,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) skip_gc: f2fs_info(sbi, "%s: call f2fs_write_checkpoint(), meta: %lld, node: %lld, data: %lld", __func__, - get_pages(sbi, F2FS_DIRTY_META), - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DATA)); + get_nr_caches(sbi, F2FS_DIRTY_META), + get_nr_caches(sbi, F2FS_DIRTY_NODES), + get_nr_caches(sbi, F2FS_DIRTY_DATA)); f2fs_down_write_trace(&sbi->gc_lock, &lc); cpc.reason = CP_PAUSE; @@ -2754,9 +2766,9 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) long long skipped_write, dirty_data; f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", - get_pages(sbi, F2FS_DIRTY_META), - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DATA)); + get_nr_caches(sbi, F2FS_DIRTY_META), + get_nr_caches(sbi, F2FS_DIRTY_NODES), + get_nr_caches(sbi, F2FS_DIRTY_DATA)); start = ktime_get(); @@ -2764,26 +2776,26 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) /* we should flush all the data to keep data consistency */ do { - skipped_write = get_pages(sbi, F2FS_SKIPPED_WRITE); - dirty_data = get_pages(sbi, F2FS_DIRTY_DATA); + skipped_write = get_nr_caches(sbi, F2FS_SKIPPED_WRITE); + dirty_data = get_nr_caches(sbi, F2FS_DIRTY_DATA); sync_inodes_sb(sbi->sb); f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); f2fs_info(sbi, "sync_inode_sb done, dirty_data: %lld, %lld, " "skipped write: %lld, %lld, retry: %d", - get_pages(sbi, F2FS_DIRTY_DATA), + get_nr_caches(sbi, F2FS_DIRTY_DATA), dirty_data, - get_pages(sbi, F2FS_SKIPPED_WRITE), + get_nr_caches(sbi, F2FS_SKIPPED_WRITE), skipped_write, retry); /* * sync_inodes_sb() has retry logic, so let's check dirty_data * in prior to skipped_write in case there is no dirty data. */ - if (!get_pages(sbi, F2FS_DIRTY_DATA)) + if (!get_nr_caches(sbi, F2FS_DIRTY_DATA)) break; - if (get_pages(sbi, F2FS_SKIPPED_WRITE) == skipped_write) + if (get_nr_caches(sbi, F2FS_SKIPPED_WRITE) == skipped_write) break; } while (retry--); @@ -2791,14 +2803,14 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) writeback = ktime_get(); - if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA) || - get_pages(sbi, F2FS_SKIPPED_WRITE))) + if (unlikely(get_nr_caches(sbi, F2FS_DIRTY_DATA) || + get_nr_caches(sbi, F2FS_SKIPPED_WRITE))) f2fs_warn(sbi, "checkpoint=enable unwritten data: %lld, skipped data: %lld, retry: %d", - get_pages(sbi, F2FS_DIRTY_DATA), - get_pages(sbi, F2FS_SKIPPED_WRITE), retry); + get_nr_caches(sbi, F2FS_DIRTY_DATA), + get_nr_caches(sbi, F2FS_SKIPPED_WRITE), retry); - if (get_pages(sbi, F2FS_SKIPPED_WRITE)) - atomic_set(&sbi->nr_pages[F2FS_SKIPPED_WRITE], 0); + if (get_nr_caches(sbi, F2FS_SKIPPED_WRITE)) + atomic_set(&sbi->nr_caches[F2FS_SKIPPED_WRITE], 0); f2fs_down_write_trace(&sbi->gc_lock, &lc); f2fs_dirty_to_prefree(sbi); @@ -2830,6 +2842,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) unsigned int flags = fc->sb_flags; int err; bool need_restart_gc = false, need_stop_gc = false; + bool need_restart_wb = false, need_stop_wb = false; bool need_restart_flush = false, need_stop_flush = false; bool need_restart_discard = false, need_stop_discard = false; bool need_enable_checkpoint = false, need_disable_checkpoint = false; @@ -2864,7 +2877,8 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) if (!org_mount_opt.s_qf_names[i]) { for (j = 0; j < i; j++) kfree(org_mount_opt.s_qf_names[j]); - return -ENOMEM; + err = -ENOMEM; + goto restore_holder; } } else { org_mount_opt.s_qf_names[i] = NULL; @@ -2988,6 +3002,18 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) need_stop_gc = true; } + if (flags & SB_RDONLY) { + if (sbi->cache_thread.cache_wb_task) { + f2fs_stop_cache_wb_thread(sbi); + need_restart_wb = true; + } + } else if (!sbi->cache_thread.cache_wb_task) { + err = f2fs_start_cache_wb_thread(sbi); + if (err) + goto restore_gc; + need_stop_wb = true; + } + if (flags & SB_RDONLY) { sync_inodes_sb(sb); @@ -2995,7 +3021,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) set_sbi_flag(sbi, SBI_IS_CLOSE); err = f2fs_sync_fs(sb, 1); if (err) - goto restore_gc; + goto restore_wb; clear_sbi_flag(sbi, SBI_IS_CLOSE); } @@ -3010,7 +3036,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) } else { err = f2fs_create_flush_cmd_control(sbi); if (err) - goto restore_gc; + goto restore_wb; need_stop_flush = true; } @@ -3078,7 +3104,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | (test_opt(sbi, POSIX_ACL) ? SB_POSIXACL : 0); - adjust_pinned_area_boundary(sbi); + f2fs_adjust_pinned_area_boundary(sbi); limit_reserve_root(sbi); fc->sb_flags = (flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME); @@ -3107,6 +3133,13 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) clear_opt(sbi, FLUSH_MERGE); f2fs_destroy_flush_cmd_control(sbi, false); } +restore_wb: + if (need_restart_wb) { + if (f2fs_start_cache_wb_thread(sbi)) + f2fs_warn(sbi, "background cache writeback thread has stopped"); + } else if (need_stop_wb) { + f2fs_stop_cache_wb_thread(sbi); + } restore_gc: if (need_restart_gc) { if (f2fs_start_gc_thread(sbi)) @@ -3125,6 +3158,9 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) sbi->mount_opt = org_mount_opt; sb->s_flags = old_sb_flags; +#ifdef CONFIG_QUOTA +restore_holder: +#endif sbi->umount_lock_holder = NULL; return err; } @@ -3305,15 +3341,42 @@ static qsize_t *f2fs_get_reserved_space(struct inode *inode) return &F2FS_I(inode)->i_reserved_quota; } +/* + * Quota files only grow and there are at most MAXQUOTAS of them, so there is + * no point in keeping them inline. Writeback of quota files does not handle + * inline data and would drop what was written to them. + */ +static int f2fs_quota_convert_inline(struct inode *inode) +{ + int err; + + if (!f2fs_has_inline_data(inode)) + return 0; + + inode_lock(inode); + err = f2fs_convert_inline_inode(inode); + inode_unlock(inode); + return err; +} + static int f2fs_quota_on_mount(struct f2fs_sb_info *sbi, int type) { + int err; + if (is_set_ckpt_flags(sbi, CP_QUOTA_NEED_FSCK_FLAG)) { f2fs_err(sbi, "quota sysfile may be corrupted, skip loading it"); return 0; } - return dquot_quota_on_mount(sbi->sb, F2FS_OPTION(sbi).s_qf_names[type], + err = dquot_quota_on_mount(sbi->sb, F2FS_OPTION(sbi).s_qf_names[type], F2FS_OPTION(sbi).s_jquota_fmt, type); + if (err) + return err; + + err = f2fs_quota_convert_inline(sb_dqopt(sbi->sb)->files[type]); + if (err) + dquot_quota_off(sbi->sb, type); + return err; } int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly) @@ -3518,6 +3581,10 @@ static int f2fs_quota_on(struct super_block *sb, int type, int format_id, inode = d_inode(path->dentry); + err = f2fs_quota_convert_inline(inode); + if (err) + goto out; + err = filemap_fdatawrite(inode->i_mapping); if (err) goto out; @@ -3741,7 +3808,7 @@ static const struct super_operations f2fs_sops = { #endif .evict_inode = f2fs_evict_inode, .put_super = f2fs_put_super, - .sync_fs = f2fs_sync_fs, + .sync_fs = f2fs_sync_fs_super, .freeze_fs = f2fs_freeze, .unfreeze_fs = f2fs_unfreeze, .statfs = f2fs_statfs, @@ -3871,13 +3938,13 @@ static const struct export_operations f2fs_export_ops = { .get_parent = f2fs_get_parent, }; -loff_t max_file_blocks(struct inode *inode) +loff_t max_file_blocks(struct f2fs_sb_info *sbi, struct inode *inode) { loff_t result = 0; loff_t leaf_count; /* - * note: previously, result is equal to (DEF_ADDRS_PER_INODE - + * note: previously, result is equal to (DEF_ADDRS_PER_INODE(sbi) - * DEFAULT_INLINE_XATTR_ADDRS), but now f2fs try to reserve more * space in inode.i_addr, it will be more safe to reassign * result as zero. @@ -3886,17 +3953,17 @@ loff_t max_file_blocks(struct inode *inode) if (inode && f2fs_compressed_file(inode)) leaf_count = ADDRS_PER_BLOCK(inode); else - leaf_count = DEF_ADDRS_PER_BLOCK; + leaf_count = DEF_ADDRS_PER_BLOCK(sbi); /* two direct node blocks */ result += (leaf_count * 2); /* two indirect node blocks */ - leaf_count *= NIDS_PER_BLOCK; + leaf_count *= NIDS_PER_BLOCK(sbi); result += (leaf_count * 2); /* one double indirect node block */ - leaf_count *= NIDS_PER_BLOCK; + leaf_count *= NIDS_PER_BLOCK(sbi); result += leaf_count; /* @@ -3905,7 +3972,8 @@ loff_t max_file_blocks(struct inode *inode) * fit within U32_MAX + 1 data units. */ - result = umin(result, F2FS_BYTES_TO_BLK(((loff_t)U32_MAX + 1) * 4096)); + result = umin(result, F2FS_BYTES_TO_BLK(sbi, + ((loff_t)U32_MAX + 1) * 4096)); return result; } @@ -3931,7 +3999,7 @@ static int __f2fs_commit_super(struct f2fs_sb_info *sbi, struct folio *folio, bio = bio_alloc(sbi->sb->s_bdev, 1, opf, GFP_NOFS); /* it doesn't need to set crypto context for superblock update */ - bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(folio->index); + bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, folio->index); if (!bio_add_folio(bio, folio, folio_size(folio), 0)) f2fs_bug_on(sbi, 1); @@ -4016,11 +4084,23 @@ static inline bool sanity_check_area_boundary(struct f2fs_sb_info *sbi, /* fix in-memory information all the time */ raw_super->segment_count = cpu_to_le32((main_end_blkaddr - segment0_blkaddr) >> log_blocks_per_seg); + if (__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_SB_CHKSUM)) { + __u32 crc = f2fs_crc32(raw_super, + offsetof(struct f2fs_super_block, crc)); + + raw_super->crc = cpu_to_le32(crc); + } if (f2fs_readonly(sb) || f2fs_hw_is_readonly(sbi)) { set_sbi_flag(sbi, SBI_NEED_SB_WRITE); res = "internally"; } else { + /* + * __f2fs_commit_super() will access log_blocksize + * in SECTOR_FROM_BLOCK(), init it in advance. + */ + sbi->log_blocksize = + le32_to_cpu(raw_super->log_blocksize); err = __f2fs_commit_super(sbi, folio, index, false); res = err ? "failed" : "done"; } @@ -4065,10 +4145,10 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, } /* only support block_size equals to PAGE_SIZE */ - if (le32_to_cpu(raw_super->log_blocksize) != F2FS_BLKSIZE_BITS) { + if (le32_to_cpu(raw_super->log_blocksize) != PAGE_SHIFT) { f2fs_info(sbi, "Invalid log_blocksize (%u), supports only %u", le32_to_cpu(raw_super->log_blocksize), - F2FS_BLKSIZE_BITS); + PAGE_SHIFT); return -EFSCORRUPTED; } @@ -4137,6 +4217,12 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, return -EFSCORRUPTED; } + if (__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_DEVICE_ALIAS) && + (!RDEV(0).path[0] || !RDEV(1).path[0])) { + f2fs_info(sbi, "Device aliasing requires a multi-device configuration"); + return -EFSCORRUPTED; + } + if (RDEV(0).path[0]) { block_t dev_seg_count = le32_to_cpu(RDEV(0).total_segments); int i = 1; @@ -4218,6 +4304,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) block_t avail_node_count, valid_node_count; unsigned int nat_blocks, nat_bits_bytes, nat_bits_blocks; unsigned int sit_blk_cnt; + unsigned long long ckpt_size, sit_end, nat_end; int i, j; total = le32_to_cpu(raw_super->segment_count); @@ -4276,7 +4363,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) return 1; if (f2fs_sb_has_readonly(sbi)) - goto check_data; + continue; for (j = i + 1; j < NR_CURSEG_NODE_TYPE; j++) { if (le32_to_cpu(ckpt->cur_node_segno[i]) == @@ -4288,14 +4375,14 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) } } } -check_data: + for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) { if (le32_to_cpu(ckpt->cur_data_segno[i]) >= main_segs || le16_to_cpu(ckpt->cur_data_blkoff[i]) >= blocks_per_seg) return 1; if (f2fs_sb_has_readonly(sbi)) - goto skip_cross; + continue; for (j = i + 1; j < NR_CURSEG_DATA_TYPE; j++) { if (le32_to_cpu(ckpt->cur_data_segno[i]) == @@ -4307,6 +4394,9 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) } } } + if (f2fs_sb_has_readonly(sbi)) + goto skip_cross; + for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { for (j = 0; j < NR_CURSEG_DATA_TYPE; j++) { if (le32_to_cpu(ckpt->cur_node_segno[i]) == @@ -4329,7 +4419,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) return 1; } - sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK); + sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK(sbi)); if (sit_bitmap_size * 8 < sit_blk_cnt) { f2fs_err(sbi, "Wrong bitmap size: sit: %u, sit_blk_cnt:%u", sit_bitmap_size, sit_blk_cnt); @@ -4346,6 +4436,29 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) return 1; } + /* + * Nothing above ties the bitmap lengths to sbi->ckpt, which is + * (1 + cp_payload) blocks. Bound both ends the way __bitmap_ptr() + * lays them out. + */ + ckpt_size = (unsigned long long)(cp_payload + 1) * F2FS_BLKSIZE(sbi); + if (__is_set_ckpt_flags(ckpt, CP_LARGE_NAT_BITMAP_FLAG)) { + nat_end = CP_MIN_CHKSUM_OFFSET + sizeof(__le32) + nat_bitmap_size; + sit_end = nat_end + sit_bitmap_size; + } else if (cp_payload > 0) { + nat_end = CP_MIN_CHKSUM_OFFSET + nat_bitmap_size; + sit_end = F2FS_BLKSIZE(sbi) + sit_bitmap_size; + } else { + sit_end = CP_MIN_CHKSUM_OFFSET + sit_bitmap_size; + nat_end = sit_end + nat_bitmap_size; + } + + if (sit_end > ckpt_size || nat_end > ckpt_size) { + f2fs_err(sbi, "Bitmaps do not fit the checkpoint pack: sit ends at %llu, nat at %llu, pack is %llu bytes", + sit_end, nat_end, ckpt_size); + return 1; + } + if (__is_set_ckpt_flags(ckpt, CP_LARGE_NAT_BITMAP_FLAG) && le32_to_cpu(ckpt->checksum_offset) != CP_MIN_CHKSUM_OFFSET) { f2fs_warn(sbi, "using deprecated layout of large_nat_bitmap, " @@ -4357,7 +4470,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) nat_blocks = nat_segs << log_blocks_per_seg; nat_bits_bytes = nat_blocks / BITS_PER_BYTE; - nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8); + nat_bits_blocks = F2FS_BLK_ALIGN(sbi, (nat_bits_bytes << 1) + 8); if (__is_set_ckpt_flags(ckpt, CP_NAT_BITS_FLAG) && (cp_payload + F2FS_CP_PACKS + NR_CURSEG_PERSIST_TYPE + nat_bits_blocks >= blocks_per_seg)) { @@ -4382,6 +4495,22 @@ static void init_sb_info(struct f2fs_sb_info *sbi) le32_to_cpu(raw_super->log_sectors_per_block); sbi->log_blocksize = le32_to_cpu(raw_super->log_blocksize); sbi->blocksize = BIT(sbi->log_blocksize); + sbi->nat_entries_per_block = sbi->blocksize / + sizeof(struct f2fs_nat_entry); + sbi->addrs_per_inode = F2FS_DEF_ADDRS_PER_INODE(sbi->blocksize); + sbi->addrs_per_block = (sbi->blocksize - + sizeof(struct node_footer)) / sizeof(__le32); + sbi->nids_per_block = sbi->addrs_per_block; + sbi->sit_entries_per_block = sbi->blocksize / + sizeof(struct f2fs_sit_entry); + sbi->orphans_per_block = (sbi->blocksize - + sizeof(struct f2fs_orphan_footer)) / sizeof(__le32); + sbi->dentries_per_block = (BITS_PER_BYTE * sbi->blocksize) / + ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1); + sbi->dentry_bitmap_size = DIV_ROUND_UP(sbi->dentries_per_block, + BITS_PER_BYTE); + sbi->dentry_reserved_size = sbi->blocksize - sbi->dentry_bitmap_size - + (SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * sbi->dentries_per_block; sbi->log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg); sbi->blocks_per_seg = BIT(sbi->log_blocks_per_seg); sbi->segs_per_sec = le32_to_cpu(raw_super->segs_per_sec); @@ -4389,12 +4518,10 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->total_sections = le32_to_cpu(raw_super->section_count); sbi->total_node_count = SEGS_TO_BLKS(sbi, ((le32_to_cpu(raw_super->segment_count_nat) / 2) * - NAT_ENTRY_PER_BLOCK)); + NAT_ENTRY_PER_BLOCK(sbi))); sbi->allocate_section_hint = le32_to_cpu(raw_super->section_count); sbi->allocate_section_policy = ALLOCATE_FORWARD_NOHINT; F2FS_ROOT_INO(sbi) = le32_to_cpu(raw_super->root_ino); - F2FS_NODE_INO(sbi) = le32_to_cpu(raw_super->node_ino); - F2FS_META_INO(sbi) = le32_to_cpu(raw_super->meta_ino); sbi->cur_victim_sec = NULL_SECNO; sbi->gc_mode = GC_NORMAL; sbi->next_victim_seg[BG_GC] = NULL_SEGNO; @@ -4436,7 +4563,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi) clear_sbi_flag(sbi, SBI_NEED_FSCK); for (i = 0; i < NR_COUNT_TYPE; i++) - atomic_set(&sbi->nr_pages[i], 0); + atomic_set(&sbi->nr_caches[i], 0); for (i = 0; i < META; i++) atomic_set(&sbi->wb_sync_req[i], 0); @@ -4490,7 +4617,7 @@ static int f2fs_report_zone_cb(struct blk_zone *zone, unsigned int idx, { struct f2fs_report_zones_args *rz_args = data; block_t unusable_blocks = (zone->len - zone->capacity) >> - F2FS_LOG_SECTORS_PER_BLOCK; + F2FS_LOG_SECTORS_PER_BLOCK(rz_args->sbi); if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) return 0; @@ -4533,10 +4660,10 @@ static int init_blkz_info(struct f2fs_sb_info *sbi, int devi) zone_sectors = bdev_zone_sectors(bdev); if (sbi->blocks_per_blkz && sbi->blocks_per_blkz != - SECTOR_TO_BLOCK(zone_sectors)) + SECTOR_TO_BLOCK(sbi, zone_sectors)) return -EINVAL; - sbi->blocks_per_blkz = SECTOR_TO_BLOCK(zone_sectors); - FDEV(devi).nr_blkz = div_u64(SECTOR_TO_BLOCK(nr_sectors), + sbi->blocks_per_blkz = SECTOR_TO_BLOCK(sbi, zone_sectors); + FDEV(devi).nr_blkz = div_u64(SECTOR_TO_BLOCK(sbi, nr_sectors), sbi->blocks_per_blkz); if (nr_sectors & (zone_sectors - 1)) FDEV(devi).nr_blkz++; @@ -5045,7 +5172,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi) { struct inode *root = d_inode(sbi->sb->s_root); struct f2fs_dir_entry *de; - struct folio *folio; + void *dentry_block = NULL; int i; if (!f2fs_sb_has_device_alias(sbi)) @@ -5060,7 +5187,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi) qstr.name = name; qstr.len = strlen(name); - de = f2fs_find_entry(root, &qstr, &folio); + de = f2fs_find_entry(root, &qstr, &dentry_block); if (!de) continue; @@ -5070,7 +5197,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi) FDEV(i).has_alias = true; iput(inode); } - f2fs_folio_put(folio, 0); + f2fs_put_dentry_block(dentry_block, false); } } @@ -5125,12 +5252,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) } mutex_init(&sbi->flush_lock); - /* set a block size */ - if (unlikely(!sb_set_blocksize(sb, F2FS_BLKSIZE))) { - f2fs_err(sbi, "unable to set blocksize"); - goto free_sbi; - } - err = read_raw_super_block(sbi, &raw_super, &valid_super_block, &recovery); if (err) @@ -5138,7 +5259,16 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_fs_info = sbi; sbi->raw_super = raw_super; + init_sb_info(sbi); + + /* set a block size */ + if (unlikely(!sb_set_blocksize(sb, sbi->blocksize))) { + f2fs_err(sbi, "unable to set blocksize %u", sbi->blocksize); + err = -EINVAL; + goto free_sb_buf; + } sbi->max_atc_write_bio_size = UINT_MAX; + sbi->max_atc_write_bio_entry_cnt = UINT_MAX; INIT_WORK(&sbi->s_error_work, f2fs_record_error_work); memcpy(sbi->errors, raw_super->s_errors, MAX_F2FS_ERRORS); @@ -5161,7 +5291,7 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto free_options; - sb->s_maxbytes = max_file_blocks(NULL) << + sb->s_maxbytes = max_file_blocks(sbi, NULL) << le32_to_cpu(raw_super->log_blocksize); sb->s_max_links = F2FS_LINK_MAX; @@ -5217,8 +5347,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto free_bio_info; - init_sb_info(sbi); - err = f2fs_init_iostat(sbi); if (err) goto free_bio_info; @@ -5231,18 +5359,14 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto free_percpu; - /* get an inode for meta space */ - sbi->meta_inode = f2fs_iget(sb, F2FS_META_INO(sbi)); - if (IS_ERR(sbi->meta_inode)) { - f2fs_err(sbi, "Failed to read F2FS meta data inode"); - err = PTR_ERR(sbi->meta_inode); - goto free_page_array_cache; - } + f2fs_init_cache(sbi, META_CACHE(sbi), F2FS_META_CACHE); + f2fs_init_cache(sbi, NODE_CACHE(sbi), F2FS_NODE_CACHE); + f2fs_init_cache(sbi, COMPRESS_CACHE(sbi), F2FS_COMPRESS_CACHE); err = f2fs_get_valid_checkpoint(sbi); if (err) { f2fs_err(sbi, "Failed to get valid F2FS checkpoint"); - goto free_meta_inode; + goto free_compress_cache; } if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_QUOTA_NEED_FSCK_FLAG)) @@ -5288,6 +5412,8 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) f2fs_init_fsync_node_info(sbi); + f2fs_init_compress_cache_context(sbi); + /* setup checkpoint request control and start checkpoint issue thread */ f2fs_init_ckpt_req_control(sbi); if (!f2fs_readonly(sb) && !test_opt(sbi, DISABLE_CHECKPOINT) && @@ -5321,7 +5447,7 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) /* get segno of first zoned block device */ sbi->first_seq_zone_segno = get_first_seq_zone_segno(sbi); - adjust_pinned_area_boundary(sbi); + f2fs_adjust_pinned_area_boundary(sbi); sbi->reserved_pin_section = f2fs_sb_has_blkzoned(sbi) ? ZONED_PIN_SEC_REQUIRED_COUNT : @@ -5339,42 +5465,30 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto free_nm; - /* get an inode for node space */ - sbi->node_inode = f2fs_iget(sb, F2FS_NODE_INO(sbi)); - if (IS_ERR(sbi->node_inode)) { - f2fs_err(sbi, "Failed to read node inode"); - err = PTR_ERR(sbi->node_inode); - goto free_stats; - } - /* read root inode and dentry */ root = f2fs_iget(sb, F2FS_ROOT_INO(sbi)); if (IS_ERR(root)) { f2fs_err(sbi, "Failed to read root inode"); err = PTR_ERR(root); - goto free_node_inode; + goto free_ino_entry; } if (!S_ISDIR(root->i_mode) || !root->i_blocks || !root->i_size || !root->i_nlink) { iput(root); err = -EINVAL; - goto free_node_inode; + goto free_ino_entry; } generic_set_sb_d_ops(sb); sb->s_root = d_make_root(root); /* allocate root dentry */ if (!sb->s_root) { err = -ENOMEM; - goto free_node_inode; + goto free_ino_entry; } - err = f2fs_init_compress_inode(sbi); - if (err) - goto free_root_inode; - err = f2fs_register_sysfs(sbi); if (err) - goto free_compress_inode; + goto free_root_inode; sbi->umount_lock_holder = current; #ifdef CONFIG_QUOTA @@ -5491,6 +5605,12 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) goto sync_free_meta; } + if (!f2fs_readonly(sb)) { + err = f2fs_start_cache_wb_thread(sbi); + if (err) + goto stop_gc_thread; + } + /* recover broken superblock */ if (recovery) { err = f2fs_commit_super(sbi, true); @@ -5513,6 +5633,8 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) sbi->umount_lock_holder = NULL; return 0; +stop_gc_thread: + f2fs_stop_gc_thread(sbi); sync_free_meta: /* safe to flush all the data */ sync_filesystem(sbi->sb); @@ -5528,23 +5650,18 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) * Some dirty meta pages can be produced by f2fs_recover_orphan_inodes() * failed by EIO. Then, iput(node_inode) can trigger balance_fs_bg() * followed by f2fs_write_checkpoint() through f2fs_write_node_pages(), which - * falls into an infinite loop in f2fs_sync_meta_pages(). + * falls into an infinite loop in f2fs_sync_meta_caches(). */ - truncate_inode_pages_final(META_MAPPING(sbi)); + f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX); /* evict some inodes being cached by GC */ evict_inodes(sb); f2fs_unregister_sysfs(sbi); -free_compress_inode: - f2fs_destroy_compress_inode(sbi); free_root_inode: dput(sb->s_root); sb->s_root = NULL; -free_node_inode: +free_ino_entry: f2fs_release_ino_entry(sbi, true); - truncate_inode_pages_final(NODE_MAPPING(sbi)); - iput(sbi->node_inode); - sbi->node_inode = NULL; -free_stats: + f2fs_truncate_node_caches(sbi, 0, ULONG_MAX); f2fs_destroy_stats(sbi); free_nm: /* stop discard thread before destroying node manager */ @@ -5560,11 +5677,10 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) free_devices: destroy_device_list(sbi); kvfree(sbi->ckpt); -free_meta_inode: - make_bad_inode(sbi->meta_inode); - iput(sbi->meta_inode); - sbi->meta_inode = NULL; -free_page_array_cache: +free_compress_cache: + f2fs_destroy_cache(COMPRESS_CACHE(sbi)); + f2fs_destroy_cache(NODE_CACHE(sbi)); + f2fs_destroy_cache(META_CACHE(sbi)); f2fs_destroy_page_array_cache(sbi); free_percpu: destroy_percpu_info(sbi); @@ -5653,7 +5769,8 @@ static void kill_f2fs_super(struct super_block *sb) * compress inode cache. */ if (test_opt(sbi, COMPRESS_CACHE)) - truncate_inode_pages_final(COMPRESS_MAPPING(sbi)); + f2fs_invalidate_compress_pages_range(sbi, + 0, UINT_MAX); #endif if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) || @@ -5699,7 +5816,7 @@ static struct file_system_type f2fs_fs_type = { .name = "f2fs", .init_fs_context = f2fs_init_fs_context, .kill_sb = kill_f2fs_super, - .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP | FS_MGTIME, }; MODULE_ALIAS_FS("f2fs"); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 811e350a1430..430356e500af 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -40,6 +40,7 @@ enum { RESERVED_BLOCKS, /* struct f2fs_sb_info */ CPRC_INFO, /* struct ckpt_req_control */ ATGC_INFO, /* struct atgc_management */ + WB_THREAD, /* struct f2fs_cache_kthread */ }; static const char *gc_mode_names[MAX_GC_MODE] = { @@ -98,6 +99,8 @@ static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) return (unsigned char *)&sbi->cprc_info; else if (struct_type == ATGC_INFO) return (unsigned char *)&sbi->am; + else if (struct_type == WB_THREAD) + return (unsigned char *)&sbi->cache_thread; return NULL; } @@ -995,6 +998,13 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } + if (!strcmp(a->attr.name, "cache_wb_interval")) { + if (t < MIN_DIRTY_CACHE_TIMEOUT || t > MAX_DIRTY_CACHE_TIMEOUT) + return -EINVAL; + sbi->cache_thread.cache_wb_interval = t; + return count; + } + __sbi_store_value(a, sbi, ptr + a->offset, t); return count; @@ -1008,7 +1018,10 @@ static ssize_t f2fs_sbi_store(struct f2fs_attr *a, bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") || a->struct_type == GC_THREAD); bool thread_entry = !strcmp(a->attr.name, "ckpt_thread_ioprio") || - !strcmp(a->attr.name, "critical_task_priority"); + !strcmp(a->attr.name, "critical_task_priority") || + !strcmp(a->attr.name, "cache_wb_interval") || + !strcmp(a->attr.name, "cache_wb_dirty_threshold") || + !strcmp(a->attr.name, "cache_wb_total_threshold"); if (gc_entry || thread_entry) { if (!down_read_trylock(&sbi->sb->s_umount)) @@ -1218,6 +1231,9 @@ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) #define ATGC_INFO_RW_ATTR(name, elname) \ F2FS_RW_ATTR(ATGC_INFO, atgc_management, name, elname) +#define WB_THREAD_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(WB_THREAD, f2fs_cache_kthread, name, elname) + /* GC_THREAD ATTR */ GC_THREAD_RW_ATTR(gc_urgent_sleep_time, urgent_sleep_time); GC_THREAD_RW_ATTR(gc_min_sleep_time, min_sleep_time); @@ -1254,7 +1270,7 @@ DCC_INFO_GENERAL_RW_ATTR(discard_io_aware); /* NM_INFO ATTR */ NM_INFO_RW_ATTR(max_roll_forward_node_blocks, max_rf_node_blocks); NM_INFO_GENERAL_RW_ATTR(ram_thresh); -NM_INFO_GENERAL_RW_ATTR(ra_nid_pages); +NM_INFO_RW_ATTR(ra_nid_pages, ra_nid_blocks); NM_INFO_GENERAL_RW_ATTR(dirty_nats_ratio); /* F2FS_SBI ATTR */ @@ -1269,6 +1285,7 @@ F2FS_SBI_RW_ATTR(umount_discard_timeout, interval_time[UMOUNT_DISCARD_TIMEOUT]); F2FS_SBI_RW_ATTR(gc_pin_file_thresh, gc_pin_file_threshold); F2FS_SBI_RW_ATTR(gc_reclaimed_segments, gc_reclaimed_segs); F2FS_SBI_RW_ATTR(max_atc_write_bio_size, max_atc_write_bio_size); +F2FS_SBI_RW_ATTR(max_atc_write_bio_entry_cnt, max_atc_write_bio_entry_cnt); F2FS_SBI_GENERAL_RW_ATTR(max_victim_search); F2FS_SBI_GENERAL_RW_ATTR(migration_granularity); F2FS_SBI_GENERAL_RW_ATTR(migration_window_granularity); @@ -1347,6 +1364,11 @@ ATGC_INFO_RW_ATTR(atgc_candidate_count, max_candidate_count); ATGC_INFO_RW_ATTR(atgc_age_weight, age_weight); ATGC_INFO_RW_ATTR(atgc_age_threshold, age_threshold); +/* WB_THREAD ATTR */ +WB_THREAD_RW_ATTR(cache_wb_interval, cache_wb_interval); +WB_THREAD_RW_ATTR(cache_wb_dirty_threshold, cache_wb_dirty_threshold); +WB_THREAD_RW_ATTR(cache_wb_total_threshold, cache_wb_total_threshold); + F2FS_GENERAL_RO_ATTR(dirty_segments); F2FS_GENERAL_RO_ATTR(free_segments); F2FS_GENERAL_RO_ATTR(ovp_segments); @@ -1404,6 +1426,8 @@ F2FS_FEATURE_RO_ATTR(linear_lookup); #endif F2FS_FEATURE_RO_ATTR(packed_ssa); F2FS_FEATURE_RO_ATTR(fserror); +F2FS_FEATURE_RO_ATTR(metadata_cache); +F2FS_FEATURE_RO_ATTR(resizable_tail); #define ATTR_LIST(name) (&f2fs_attr_##name.attr) static struct attribute *f2fs_attrs[] = { @@ -1514,6 +1538,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(gc_segment_mode), ATTR_LIST(gc_reclaimed_segments), ATTR_LIST(max_atc_write_bio_size), + ATTR_LIST(max_atc_write_bio_entry_cnt), ATTR_LIST(max_fragment_chunk), ATTR_LIST(max_fragment_hole), ATTR_LIST(current_atomic_write), @@ -1533,6 +1558,9 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(lock_duration_priority), ATTR_LIST(adjust_lock_priority), ATTR_LIST(critical_task_priority), + ATTR_LIST(cache_wb_interval), + ATTR_LIST(cache_wb_dirty_threshold), + ATTR_LIST(cache_wb_total_threshold), NULL, }; ATTRIBUTE_GROUPS(f2fs); @@ -1574,6 +1602,8 @@ static struct attribute *f2fs_feat_attrs[] = { #endif BASE_ATTR_LIST(packed_ssa), BASE_ATTR_LIST(fserror), + BASE_ATTR_LIST(metadata_cache), + BASE_ATTR_LIST(resizable_tail), NULL, }; ATTRIBUTE_GROUPS(f2fs_feat); @@ -1871,8 +1901,8 @@ static int __maybe_unused disk_map_seq_show(struct seq_file *seq, struct f2fs_sb_info *sbi = F2FS_SB(sb); int i; - seq_printf(seq, "Address Layout : %5luB Block address (# of Segments)\n", - F2FS_BLKSIZE); + seq_printf(seq, "Address Layout : %5uB Block address (# of Segments)\n", + F2FS_BLKSIZE(sbi)); seq_printf(seq, " SB : %12s\n", "0/1024B"); seq_printf(seq, " seg0_blkaddr : 0x%010x\n", SEG0_BLKADDR(sbi)); seq_printf(seq, " Checkpoint : 0x%010x (%10d)\n", @@ -1889,13 +1919,13 @@ static int __maybe_unused disk_map_seq_show(struct seq_file *seq, seq_printf(seq, " Main : 0x%010x (%10d)\n", SM_I(sbi)->main_blkaddr, le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_main)); - seq_printf(seq, " Block size : %12lu KB\n", F2FS_BLKSIZE >> 10); + seq_printf(seq, " Block size : %12u KB\n", F2FS_BLKSIZE(sbi) >> 10); seq_printf(seq, " Segment size : %12d MB\n", - (BLKS_PER_SEG(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10); + (BLKS_PER_SEG(sbi) << (F2FS_BLKSIZE_BITS(sbi) - 10)) >> 10); seq_printf(seq, " Segs/Sections : %12d\n", SEGS_PER_SEC(sbi)); seq_printf(seq, " Section size : %12d MB\n", - (BLKS_PER_SEC(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10); + (BLKS_PER_SEC(sbi) << (F2FS_BLKSIZE_BITS(sbi) - 10)) >> 10); seq_printf(seq, " # of Sections : %12d\n", le32_to_cpu(F2FS_RAW_SUPER(sbi)->section_count)); diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index f3a4617656db..4a2cea4288f9 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -47,7 +47,8 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, struct address_space *mapping = inode->i_mapping; const struct address_space_operations *aops = mapping->a_ops; - if (pos + count > F2FS_BLK_TO_BYTES(max_file_blocks(inode))) + if (pos + count > F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), + max_file_blocks(F2FS_I_SB(inode), inode))) return -EFBIG; while (count) { @@ -211,7 +212,8 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, /* Get the descriptor */ if (pos + size < pos || - pos + size > F2FS_BLK_TO_BYTES(max_file_blocks(inode)) || + pos + size > F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), + max_file_blocks(F2FS_I_SB(inode), inode)) || pos < f2fs_verity_metadata_pos(inode) || size > INT_MAX) { f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr"); f2fs_handle_error(F2FS_I_SB(inode), diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index ffe4815ffa6d..0ed879fc3076 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -138,7 +138,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, #ifdef CONFIG_F2FS_FS_SECURITY static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array, - void *folio) + void *fs_data) { const struct xattr *xattr; int err = 0; @@ -146,7 +146,7 @@ static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array, for (xattr = xattr_array; xattr->name != NULL; xattr++) { err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_SECURITY, xattr->name, xattr->value, - xattr->value_len, folio, 0); + xattr->value_len, fs_data, 0); if (err < 0) break; } @@ -154,10 +154,10 @@ static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array, } int f2fs_init_security(struct inode *inode, struct inode *dir, - const struct qstr *qstr, struct folio *ifolio) + const struct qstr *qstr, struct f2fs_cached_block *ientry) { return security_inode_init_security(inode, dir, qstr, - f2fs_initxattrs, ifolio); + f2fs_initxattrs, ientry); } #endif @@ -273,25 +273,25 @@ static struct f2fs_xattr_entry *__find_inline_xattr(struct inode *inode, return entry; } -static int read_inline_xattr(struct inode *inode, struct folio *ifolio, +static int read_inline_xattr(struct inode *inode, struct f2fs_cached_block *ientry, void *txattr_addr) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); unsigned int inline_size = inline_xattr_size(inode); - struct folio *folio = NULL; + struct f2fs_cached_block *in_entry = NULL; void *inline_addr; - if (ifolio) { - inline_addr = inline_xattr_addr(inode, ifolio); + if (ientry) { + inline_addr = inline_xattr_addr(inode, ientry); } else { - folio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(folio)) - return PTR_ERR(folio); + in_entry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(in_entry)) + return PTR_ERR(in_entry); - inline_addr = inline_xattr_addr(inode, folio); + inline_addr = inline_xattr_addr(inode, in_entry); } memcpy(txattr_addr, inline_addr, inline_size); - f2fs_folio_put(folio, true); + f2fs_put_cache(in_entry, true); return 0; } @@ -301,22 +301,23 @@ static int read_xattr_block(struct inode *inode, void *txattr_addr) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); nid_t xnid = F2FS_I(inode)->i_xattr_nid; unsigned int inline_size = inline_xattr_size(inode); - struct folio *xfolio; + struct f2fs_cached_block *xentry; void *xattr_addr; /* The inode already has an extended attribute block. */ - xfolio = f2fs_get_xnode_folio(sbi, xnid); - if (IS_ERR(xfolio)) - return PTR_ERR(xfolio); + xentry = f2fs_get_xnode_cache(sbi, xnid); + if (IS_ERR(xentry)) + return PTR_ERR(xentry); - xattr_addr = folio_address(xfolio); - memcpy(txattr_addr + inline_size, xattr_addr, VALID_XATTR_BLOCK_SIZE); - f2fs_folio_put(xfolio, true); + xattr_addr = cache_address(xentry); + memcpy(txattr_addr + inline_size, xattr_addr, + VALID_XATTR_BLOCK_SIZE(inode)); + f2fs_put_cache(xentry, true); return 0; } -static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio, +static int lookup_all_xattrs(struct inode *inode, struct f2fs_cached_block *ientry, unsigned int index, unsigned int len, const char *name, struct f2fs_xattr_entry **xe, void **base_addr, int *base_size, @@ -340,7 +341,7 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio, /* read from inline xattr */ if (inline_size) { - err = read_inline_xattr(inode, ifolio, txattr_addr); + err = read_inline_xattr(inode, ientry, txattr_addr); if (err) goto out; @@ -388,12 +389,12 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio, return err; } -static int read_all_xattrs(struct inode *inode, struct folio *ifolio, +static int read_all_xattrs(struct inode *inode, struct f2fs_cached_block *ientry, void **base_addr) { struct f2fs_xattr_header *header; nid_t xnid = F2FS_I(inode)->i_xattr_nid; - unsigned int size = VALID_XATTR_BLOCK_SIZE; + unsigned int size = VALID_XATTR_BLOCK_SIZE(inode); unsigned int inline_size = inline_xattr_size(inode); void *txattr_addr; int err; @@ -405,7 +406,7 @@ static int read_all_xattrs(struct inode *inode, struct folio *ifolio, /* read from inline xattr */ if (inline_size) { - err = read_inline_xattr(inode, ifolio, txattr_addr); + err = read_inline_xattr(inode, ientry, txattr_addr); if (err) goto fail; } @@ -432,14 +433,14 @@ static int read_all_xattrs(struct inode *inode, struct folio *ifolio, } static inline int write_all_xattrs(struct inode *inode, __u32 hsize, - void *txattr_addr, struct folio *ifolio) + void *txattr_addr, struct f2fs_cached_block *ientry) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); size_t inline_size = inline_xattr_size(inode); - struct folio *in_folio = NULL; + struct f2fs_cached_block *in_entry = NULL; void *xattr_addr; void *inline_addr = NULL; - struct folio *xfolio; + struct f2fs_cached_block *xentry; nid_t new_nid = 0; int err = 0; @@ -449,75 +450,75 @@ static inline int write_all_xattrs(struct inode *inode, __u32 hsize, /* write to inline xattr */ if (inline_size) { - if (ifolio) { - inline_addr = inline_xattr_addr(inode, ifolio); + if (ientry) { + inline_addr = inline_xattr_addr(inode, ientry); } else { - in_folio = f2fs_get_inode_folio(sbi, inode->i_ino); - if (IS_ERR(in_folio)) { + in_entry = f2fs_get_inode_cache(sbi, inode->i_ino); + if (IS_ERR(in_entry)) { f2fs_alloc_nid_failed(sbi, new_nid); - return PTR_ERR(in_folio); + return PTR_ERR(in_entry); } - inline_addr = inline_xattr_addr(inode, in_folio); + inline_addr = inline_xattr_addr(inode, in_entry); } - f2fs_folio_wait_writeback(ifolio ? ifolio : in_folio, - NODE, true, true); + f2fs_cache_wait_writeback(ientry ? ientry : in_entry); /* no need to use xattr node block */ if (hsize <= inline_size) { err = f2fs_truncate_xattr_node(inode); f2fs_alloc_nid_failed(sbi, new_nid); if (err) { - f2fs_folio_put(in_folio, true); + f2fs_put_cache(in_entry, true); return err; } memcpy(inline_addr, txattr_addr, inline_size); - folio_mark_dirty(ifolio ? ifolio : in_folio); + f2fs_mark_cache_dirty(ientry ? ientry : in_entry); goto in_page_out; } } /* write to xattr node block */ if (F2FS_I(inode)->i_xattr_nid) { - xfolio = f2fs_get_xnode_folio(sbi, F2FS_I(inode)->i_xattr_nid); - if (IS_ERR(xfolio)) { - err = PTR_ERR(xfolio); + xentry = f2fs_get_xnode_cache(sbi, F2FS_I(inode)->i_xattr_nid); + if (IS_ERR(xentry)) { + err = PTR_ERR(xentry); f2fs_alloc_nid_failed(sbi, new_nid); goto in_page_out; } f2fs_bug_on(sbi, new_nid); - f2fs_folio_wait_writeback(xfolio, NODE, true, true); + f2fs_cache_wait_writeback(xentry); } else { struct dnode_of_data dn; set_new_dnode(&dn, inode, NULL, NULL, new_nid); - xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET); - if (IS_ERR(xfolio)) { - err = PTR_ERR(xfolio); + xentry = f2fs_new_node_cache(&dn, XATTR_NODE_OFFSET); + if (IS_ERR(xentry)) { + err = PTR_ERR(xentry); f2fs_alloc_nid_failed(sbi, new_nid); goto in_page_out; } f2fs_alloc_nid_done(sbi, new_nid); } - xattr_addr = folio_address(xfolio); + xattr_addr = cache_address(xentry); if (inline_size) memcpy(inline_addr, txattr_addr, inline_size); - memcpy(xattr_addr, txattr_addr + inline_size, VALID_XATTR_BLOCK_SIZE); + memcpy(xattr_addr, txattr_addr + inline_size, + VALID_XATTR_BLOCK_SIZE(inode)); if (inline_size) - folio_mark_dirty(ifolio ? ifolio : in_folio); - folio_mark_dirty(xfolio); + f2fs_mark_cache_dirty(ientry ? ientry : in_entry); + f2fs_mark_cache_dirty(xentry); - f2fs_folio_put(xfolio, true); + f2fs_put_cache(xentry, true); in_page_out: - f2fs_folio_put(in_folio, true); + f2fs_put_cache(in_entry, true); return err; } int f2fs_getxattr(struct inode *inode, int index, const char *name, - void *buffer, size_t buffer_size, struct folio *ifolio) + void *buffer, size_t buffer_size, struct f2fs_cached_block *ientry) { - struct f2fs_xattr_entry *entry = NULL; + struct f2fs_xattr_entry *xe = NULL; int error; unsigned int size, len; void *base_addr = NULL; @@ -531,16 +532,16 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name, if (len > F2FS_NAME_LEN) return -ERANGE; - if (!ifolio) + if (!ientry) f2fs_down_read(&F2FS_I(inode)->i_xattr_sem); - error = lookup_all_xattrs(inode, ifolio, index, len, name, - &entry, &base_addr, &base_size, &is_inline); - if (!ifolio) + error = lookup_all_xattrs(inode, ientry, index, len, name, + &xe, &base_addr, &base_size, &is_inline); + if (!ientry) f2fs_up_read(&F2FS_I(inode)->i_xattr_sem); if (error) return error; - size = le16_to_cpu(entry->e_value_size); + size = le16_to_cpu(xe->e_value_size); if (buffer && size > buffer_size) { error = -ERANGE; @@ -548,7 +549,7 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name, } if (buffer) { - char *pval = entry->e_name + entry->e_name_len; + char *pval = xe->e_name + xe->e_name_len; if (base_size - (pval - (char *)base_addr) < size) { error = -ERANGE; @@ -632,7 +633,7 @@ static bool f2fs_xattr_value_same(struct f2fs_xattr_entry *entry, static int __f2fs_setxattr(struct inode *inode, int index, const char *name, const void *value, size_t size, - struct folio *ifolio, int flags) + struct f2fs_cached_block *ientry, int flags) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_xattr_entry *here, *last; @@ -656,7 +657,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, if (size > MAX_VALUE_LEN(inode)) return -E2BIG; retry: - error = read_all_xattrs(inode, ifolio, &base_addr); + error = read_all_xattrs(inode, ientry, &base_addr); if (error) return error; @@ -773,7 +774,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, *(u32 *)((u8 *)last + newsize) = 0; } - error = write_all_xattrs(inode, new_hsize, base_addr, ifolio); + error = write_all_xattrs(inode, new_hsize, base_addr, ientry); if (error) goto exit; @@ -807,7 +808,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, int f2fs_setxattr(struct inode *inode, int index, const char *name, const void *value, size_t size, - struct folio *ifolio, int flags) + struct f2fs_cached_block *ientry, int flags) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_lock_context lc; @@ -823,9 +824,9 @@ int f2fs_setxattr(struct inode *inode, int index, const char *name, return err; /* this case is only from f2fs_init_inode_metadata */ - if (ifolio) + if (ientry) return __f2fs_setxattr(inode, index, name, value, - size, ifolio, flags); + size, ientry, flags); f2fs_balance_fs(sbi, true); f2fs_lock_op(sbi, &lc); @@ -848,4 +849,4 @@ int __init f2fs_init_xattr_cache(void) void f2fs_destroy_xattr_cache(void) { kmem_cache_destroy(inline_xattr_slab); -} \ No newline at end of file +} diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h index bce3d93e4755..870524b667ed 100644 --- a/fs/f2fs/xattr.h +++ b/fs/f2fs/xattr.h @@ -71,21 +71,22 @@ struct f2fs_xattr_entry { for (entry = XATTR_FIRST_ENTRY(addr);\ !IS_XATTR_LAST_ENTRY(entry);\ entry = XATTR_NEXT_ENTRY(entry)) -#define VALID_XATTR_BLOCK_SIZE (PAGE_SIZE - sizeof(struct node_footer)) +#define VALID_XATTR_BLOCK_SIZE(i) (i_blocksize(i) - \ + sizeof(struct node_footer)) #define XATTR_PADDING_SIZE (sizeof(__u32)) #define XATTR_SIZE(i) ((F2FS_I(i)->i_xattr_nid ? \ - VALID_XATTR_BLOCK_SIZE : 0) + \ + VALID_XATTR_BLOCK_SIZE(i) : 0) + \ (inline_xattr_size(i))) #define MIN_OFFSET(i) XATTR_ALIGN(inline_xattr_size(i) + \ - VALID_XATTR_BLOCK_SIZE) + VALID_XATTR_BLOCK_SIZE(i)) #define MAX_VALUE_LEN(i) (MIN_OFFSET(i) - \ sizeof(struct f2fs_xattr_header) - \ sizeof(struct f2fs_xattr_entry)) #define MIN_INLINE_XATTR_SIZE (sizeof(struct f2fs_xattr_header) / sizeof(__le32)) -#define MAX_INLINE_XATTR_SIZE \ - (DEF_ADDRS_PER_INODE - \ +#define MAX_INLINE_XATTR_SIZE(blocksize) \ + (F2FS_DEF_ADDRS_PER_INODE(blocksize) - \ F2FS_TOTAL_EXTRA_ATTR_SIZE / sizeof(__le32) - \ DEF_INLINE_RESERVED_SIZE - \ MIN_INLINE_DENTRY_SIZE / sizeof(__le32)) @@ -130,9 +131,9 @@ extern const struct xattr_handler f2fs_xattr_security_handler; extern const struct xattr_handler * const f2fs_xattr_handlers[]; int f2fs_setxattr(struct inode *, int, const char *, const void *, - size_t, struct folio *, int); + size_t, struct f2fs_cached_block *, int); int f2fs_getxattr(struct inode *, int, const char *, void *, - size_t, struct folio *); + size_t, struct f2fs_cached_block *); ssize_t f2fs_listxattr(struct dentry *, char *, size_t); int __init f2fs_init_xattr_cache(void); void f2fs_destroy_xattr_cache(void); @@ -142,13 +143,13 @@ void f2fs_destroy_xattr_cache(void); #define f2fs_listxattr NULL static inline int f2fs_setxattr(struct inode *inode, int index, const char *name, const void *value, size_t size, - struct folio *folio, int flags) + struct f2fs_cached_block *ientry, int flags) { return -EOPNOTSUPP; } static inline int f2fs_getxattr(struct inode *inode, int index, const char *name, void *buffer, - size_t buffer_size, struct folio *dfolio) + size_t buffer_size, struct f2fs_cached_block *ientry) { return -EOPNOTSUPP; } @@ -158,10 +159,10 @@ static inline void f2fs_destroy_xattr_cache(void) { } #ifdef CONFIG_F2FS_FS_SECURITY int f2fs_init_security(struct inode *, struct inode *, - const struct qstr *, struct folio *); + const struct qstr *, struct f2fs_cached_block *); #else static inline int f2fs_init_security(struct inode *inode, struct inode *dir, - const struct qstr *qstr, struct folio *ifolio) + const struct qstr *qstr, struct f2fs_cached_block *ientry) { return 0; } diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index bb2b6cd5d507..3081702b1ddb 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -14,9 +14,9 @@ #define F2FS_SUPER_OFFSET 1024 /* byte-size offset */ #define F2FS_MIN_LOG_SECTOR_SIZE 9 /* 9 bits for 512 bytes */ #define F2FS_MAX_LOG_SECTOR_SIZE PAGE_SHIFT /* Max is Block Size */ -#define F2FS_LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) /* log number for sector/blk */ -#define F2FS_BLKSIZE PAGE_SIZE /* support only block == page */ -#define F2FS_BLKSIZE_BITS PAGE_SHIFT /* bits for F2FS_BLKSIZE */ +#define F2FS_MIN_LOG_BLOCKSIZE 12 +#define F2FS_MIN_BLKSIZE 4096UL +#define F2FS_MAX_BLKSIZE PAGE_SIZE #define F2FS_MAX_EXTENSION 64 /* # of extension entries */ #define F2FS_EXTENSION_LEN 8 /* max size of extension */ @@ -24,19 +24,25 @@ #define NEW_ADDR ((block_t)-1) /* used as block_t addresses */ #define COMPRESS_ADDR ((block_t)-2) /* used as compressed data flag */ -#define F2FS_BLKSIZE_MASK (F2FS_BLKSIZE - 1) -#define F2FS_BYTES_TO_BLK(bytes) ((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS) -#define F2FS_BLK_TO_BYTES(blk) ((unsigned long long)(blk) << F2FS_BLKSIZE_BITS) -#define F2FS_BLK_END_BYTES(blk) (F2FS_BLK_TO_BYTES(blk + 1) - 1) -#define F2FS_BLK_ALIGN(x) (F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1)) +#define F2FS_BLKSIZE(sbi) ((sbi)->blocksize) +#define F2FS_BLKSIZE_BITS(sbi) ((sbi)->log_blocksize) +#define F2FS_BLKSIZE_MASK(sbi) (F2FS_BLKSIZE(sbi) - 1) +#define F2FS_LOG_SECTORS_PER_BLOCK(sbi) (F2FS_BLKSIZE_BITS(sbi) - 9) +#define F2FS_BLKS_PER_PAGE(sbi) (PAGE_SIZE / F2FS_BLKSIZE(sbi)) +#define F2FS_BYTES_TO_BLK(sbi, bytes) \ + ((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS(sbi)) +#define F2FS_BLK_TO_BYTES(sbi, blk) \ + ((unsigned long long)(blk) << F2FS_BLKSIZE_BITS(sbi)) +#define F2FS_BLK_END_BYTES(sbi, blk) \ + (F2FS_BLK_TO_BYTES(sbi, (blk) + 1) - 1) +#define F2FS_BLK_ALIGN(sbi, bytes) \ + F2FS_BYTES_TO_BLK(sbi, (unsigned long long)(bytes) + \ + F2FS_BLKSIZE(sbi) - 1) /* 0, 1(node nid), 2(meta nid) are reserved node id */ #define F2FS_RESERVED_NODE_NUM 3 #define F2FS_ROOT_INO(sbi) ((sbi)->root_ino_num) -#define F2FS_NODE_INO(sbi) ((sbi)->node_ino_num) -#define F2FS_META_INO(sbi) ((sbi)->meta_ino_num) -#define F2FS_COMPRESS_INO(sbi) (NM_I(sbi)->max_nid) #define F2FS_MAX_QUOTAS 3 @@ -214,20 +220,27 @@ struct f2fs_checkpoint { unsigned char sit_nat_version_bitmap[]; } __packed; -#define CP_CHKSUM_OFFSET (F2FS_BLKSIZE - sizeof(__le32)) /* default chksum offset in checkpoint */ #define CP_MIN_CHKSUM_OFFSET \ (offsetof(struct f2fs_checkpoint, sit_nat_version_bitmap)) /* * For orphan inode management + * + * The number of inode entries in an orphan block depends on the filesystem + * block size. Its exact on-disk layout is: + * + * 0 blocksize - 16 blocksize + * +--------------------------+--------------------------+ + * | ino[0] ... ino[n - 1] | struct f2fs_orphan_footer | + * +--------------------------+--------------------------+ + * + * n = (blocksize - sizeof(struct f2fs_orphan_footer)) / sizeof(__le32) */ -#define F2FS_ORPHANS_PER_BLOCK ((F2FS_BLKSIZE - 4 * sizeof(__le32)) / sizeof(__le32)) - -#define GET_ORPHAN_BLOCKS(n) (((n) + F2FS_ORPHANS_PER_BLOCK - 1) / \ - F2FS_ORPHANS_PER_BLOCK) - struct f2fs_orphan_block { - __le32 ino[F2FS_ORPHANS_PER_BLOCK]; /* inode numbers */ + DECLARE_FLEX_ARRAY(__le32, ino); +} __packed; + +struct f2fs_orphan_footer { __le32 reserved; /* reserved */ __le16 blk_addr; /* block index in current CP */ __le16 blk_count; /* Number of orphan inode blocks in CP */ @@ -260,26 +273,14 @@ struct node_footer { } __packed; /* Address Pointers in an Inode */ -#define DEF_ADDRS_PER_INODE ((F2FS_BLKSIZE - OFFSET_OF_END_OF_I_EXT \ - - SIZE_OF_I_NID \ - - sizeof(struct node_footer)) / sizeof(__le32)) -#define CUR_ADDRS_PER_INODE(inode) (DEF_ADDRS_PER_INODE - \ - get_extra_isize(inode)) +#define F2FS_DEF_ADDRS_PER_INODE(blocksize) \ + (((blocksize) - OFFSET_OF_END_OF_I_EXT - SIZE_OF_I_NID - \ + sizeof(struct node_footer)) / sizeof(__le32)) #define DEF_NIDS_PER_INODE 5 /* Node IDs in an Inode */ #define ADDRS_PER_INODE(inode) addrs_per_page(inode, true) /* Address Pointers in a Direct Block */ -#define DEF_ADDRS_PER_BLOCK ((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32)) #define ADDRS_PER_BLOCK(inode) addrs_per_page(inode, false) -/* Node IDs in an Indirect Block */ -#define NIDS_PER_BLOCK ((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32)) - -#define ADDRS_PER_PAGE(folio, inode) (addrs_per_page(inode, IS_INODE(folio))) - -#define NODE_DIR1_BLOCK (DEF_ADDRS_PER_INODE + 1) -#define NODE_DIR2_BLOCK (DEF_ADDRS_PER_INODE + 2) -#define NODE_IND1_BLOCK (DEF_ADDRS_PER_INODE + 3) -#define NODE_IND2_BLOCK (DEF_ADDRS_PER_INODE + 4) -#define NODE_DIND_BLOCK (DEF_ADDRS_PER_INODE + 5) +#define ADDRS_PER_PAGE(folio, inode) (addrs_per_page(inode, IS_INODE(F2FS_I_SB(inode), folio))) #define F2FS_INLINE_XATTR 0x01 /* file inline xattr flag */ #define F2FS_INLINE_DATA 0x02 /* file inline data flag */ @@ -339,18 +340,26 @@ struct f2fs_inode { */ __le32 i_extra_end[0]; /* for attribute size calculation */ } __packed; - __le32 i_addr[DEF_ADDRS_PER_INODE]; /* Pointers to data blocks */ + DECLARE_FLEX_ARRAY(__le32, i_addr); /* data block pointers */ }; - __le32 i_nid[DEF_NIDS_PER_INODE]; /* direct(2), indirect(2), - double_indirect(1) node id */ + /* + * __le32 i_nid[DEF_NIDS_PER_INODE]; + * direct(2), indirect(2), double_indirect(1) node IDs + * + * It is stored immediately before the node footer at the end of the + * filesystem block. Its offset depends on the filesystem block size, so + * locate it dynamically with F2FS_INODE_NIDS(). + */ } __packed; struct direct_node { - __le32 addr[DEF_ADDRS_PER_BLOCK]; /* array of data block address */ + /* The address count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(__le32, addr); /* array of data block address */ } __packed; struct indirect_node { - __le32 nid[NIDS_PER_BLOCK]; /* array of data block address */ + /* The node ID count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(__le32, nid); /* array of data block address */ } __packed; enum { @@ -369,14 +378,18 @@ struct f2fs_node { struct direct_node dn; struct indirect_node in; }; - struct node_footer footer; + /* + * struct node_footer footer; + * + * It is stored at the end of the filesystem block, after the inode or + * direct/indirect node data. Its offset depends on the filesystem block + * size, so locate it dynamically with F2FS_NODE_FOOTER(). + */ } __packed; /* * For NAT entries */ -#define NAT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_nat_entry)) - struct f2fs_nat_entry { __u8 version; /* latest version of cached nat entry */ __le32 ino; /* inode number */ @@ -384,7 +397,8 @@ struct f2fs_nat_entry { } __packed; struct f2fs_nat_block { - struct f2fs_nat_entry entries[NAT_ENTRY_PER_BLOCK]; + /* The entry count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(struct f2fs_nat_entry, entries); } __packed; /* @@ -396,8 +410,6 @@ struct f2fs_nat_block { * Not allow to change this. */ #define SIT_VBLOCK_MAP_SIZE 64 -#define SIT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_sit_entry)) - /* * F2FS uses 4 bytes to represent block address. As a result, supported size of * disk is 16 TB for a 4K page size and 64 TB for a 16K page size and it equals @@ -424,8 +436,13 @@ struct f2fs_sit_entry { __le64 mtime; /* segment age for cleaning */ } __packed; +/* + * The on-disk SIT block is a filesystem-block-sized array of SIT entries. + * Its entry count depends on the filesystem block size, so it must be + * calculated by the caller rather than implied by this C structure. + */ struct f2fs_sit_block { - struct f2fs_sit_entry entries[SIT_ENTRY_PER_BLOCK]; + DECLARE_FLEX_ARRAY(struct f2fs_sit_entry, entries); } __packed; /* @@ -595,15 +612,7 @@ typedef __le32 f2fs_hash_t; * dentry, when converting inline dentry we should handle this carefully. */ -/* the number of dentry in a block */ -#define NR_DENTRY_IN_BLOCK ((BITS_PER_BYTE * F2FS_BLKSIZE) / \ - ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1)) #define SIZE_OF_DIR_ENTRY 11 /* by byte */ -#define SIZE_OF_DENTRY_BITMAP ((NR_DENTRY_IN_BLOCK + BITS_PER_BYTE - 1) / \ - BITS_PER_BYTE) -#define SIZE_OF_RESERVED (F2FS_BLKSIZE - ((SIZE_OF_DIR_ENTRY + \ - F2FS_SLOT_LEN) * \ - NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP)) #define MIN_INLINE_DENTRY_SIZE 40 /* just include '.' and '..' entries */ /* One directory entry slot representing F2FS_SLOT_LEN-sized file name */ @@ -614,14 +623,21 @@ struct f2fs_dir_entry { __u8 file_type; /* file type */ } __packed; -/* Block-sized directory entry block */ -struct f2fs_dentry_block { - /* validity bitmap for directory entries in each block */ - __u8 dentry_bitmap[SIZE_OF_DENTRY_BITMAP]; - __u8 reserved[SIZE_OF_RESERVED]; - struct f2fs_dir_entry dentry[NR_DENTRY_IN_BLOCK]; - __u8 filename[NR_DENTRY_IN_BLOCK][F2FS_SLOT_LEN]; -} __packed; +/* + * A dentry block is laid out as follows, where the number of entries and all + * offsets are determined by the filesystem block size at runtime: + * + * 0 blocksize + * +--------+----------+-------------------+-----------------------+ + * | bitmap | reserved | dir_entry[entries]| filename[entries][8] | + * +--------+----------+-------------------+-----------------------+ + * + * entries = (BITS_PER_BYTE * blocksize) / + * ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1) + * bitmap_size = DIV_ROUND_UP(entries, BITS_PER_BYTE) + * reserved_size = blocksize - bitmap_size - + * (SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * entries + */ #define F2FS_DEF_PROJID 0 /* default project ID */ diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index d53be932df01..df4cd346ae72 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -1430,6 +1430,50 @@ DEFINE_EVENT(f2fs__folio, f2fs_set_page_dirty, TP_ARGS(folio, type) ); +DECLARE_EVENT_CLASS(f2fs__cached_block, + + TP_PROTO(struct f2fs_cached_block *block, int type), + + TP_ARGS(block, type), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(pgoff_t, index) + __field(int, type) + __field(int, dirty) + __field(int, uptodate) + ), + + TP_fast_assign( + __entry->dev = block->cache->sbi->sb->s_dev; + __entry->index = block->index; + __entry->type = type; + __entry->dirty = f2fs_cache_test_dirty(block); + __entry->uptodate = f2fs_cache_test_uptodate(block); + ), + + TP_printk("dev = (%d,%d), %s, index = %lu, dirty = %d, uptodate = %d", + show_dev(__entry->dev), + show_block_type(__entry->type), + (unsigned long)__entry->index, + __entry->dirty, + __entry->uptodate) +); + +DEFINE_EVENT(f2fs__cached_block, f2fs_write_cache, + + TP_PROTO(struct f2fs_cached_block *block, int type), + + TP_ARGS(block, type) +); + +DEFINE_EVENT(f2fs__cached_block, f2fs_cache_set_dirty, + + TP_PROTO(struct f2fs_cached_block *block, int type), + + TP_ARGS(block, type) +); + TRACE_EVENT(f2fs_replace_atomic_write_block, TP_PROTO(struct inode *inode, struct inode *cow_inode, pgoff_t index, @@ -1574,6 +1618,33 @@ TRACE_EVENT(f2fs_writepages, __entry->for_sync) ); +TRACE_EVENT(f2fs_write_caches, + + TP_PROTO(struct f2fs_sb_info *sbi, long nr_to_write, long nwritten, int type), + + TP_ARGS(sbi, nr_to_write, nwritten, type), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(long, nr_to_write) + __field(long, nwritten) + __field(int, type) + ), + + TP_fast_assign( + __entry->dev = sbi->sb->s_dev; + __entry->nr_to_write = nr_to_write; + __entry->nwritten = nwritten; + __entry->type = type; + ), + + TP_printk("dev = (%d,%d), %s, nr_to_write = %ld, nwritten = %ld", + show_dev(__entry->dev), + show_block_type(__entry->type), + __entry->nr_to_write, + __entry->nwritten) +); + TRACE_EVENT(f2fs_readpages, TP_PROTO(struct inode *inode, pgoff_t start, unsigned int nrpage),