github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-29 10:41:49 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

btrfs: cache folio size and shift in extent_buffer

Browse Source 
After the conversion to folio interfaces (but without the patch to
enable larger folio allocation), there is an LTP report about observable
performance drop on metadata heavy operations.

https://lore.kernel.org/linux-btrfs/202312221750.571925bd-oliver.sang@intel.com/

This drop is caused by the extra code of calculating the
folio_size()/folio_shift(), instead of the old hard coded
PAGE_SIZE/PAGE_SHIFT.

To slightly reduce the overhead, just cache both folio_size and
folio_shift in extent_buffer.

The two new members (u32 folio_size and u8 folio_shift) are stored
inside the holes of extent_buffer. folio_size is shared with len, which
is reduced to u32. The size of eb does not change.

Signed-off-by: Qu Wenruo <wqu@suse.com>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>
This commit is contained in:
Qu Wenruo 2024-01-05 16:05:55 +10:30 committed by David Sterba
parent 4d02b54390
commit 84cda1a608
5 changed files with 42 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fs/btrfs/accessors.c
View File

@ -63,8 +63,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \
const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \ const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \
const unsigned long oil = get_eb_offset_in_folio(token->eb, \ const unsigned long oil = get_eb_offset_in_folio(token->eb, \
member_offset);\ member_offset);\
const int unit_size = folio_size(token->eb->folios[0]); \ const int unit_size = token->eb->folio_size; \
const int unit_shift = folio_shift(token->eb->folios[0]); \ const int unit_shift = token->eb->folio_shift; \
const int size = sizeof(u##bits); \ const int size = sizeof(u##bits); \
u8 lebytes[sizeof(u##bits)]; \ u8 lebytes[sizeof(u##bits)]; \
const int part = unit_size - oil; \ const int part = unit_size - oil; \
@ -94,7 +94,7 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \
const unsigned long idx = get_eb_folio_index(eb, member_offset);\ const unsigned long idx = get_eb_folio_index(eb, member_offset);\
const unsigned long oil = get_eb_offset_in_folio(eb, \ const unsigned long oil = get_eb_offset_in_folio(eb, \
member_offset);\ member_offset);\
const int unit_size = folio_size(eb->folios[0]); \ const int unit_size = eb->folio_size; \
char *kaddr = folio_address(eb->folios[idx]); \ char *kaddr = folio_address(eb->folios[idx]); \
const int size = sizeof(u##bits); \ const int size = sizeof(u##bits); \
const int part = unit_size - oil; \ const int part = unit_size - oil; \
@ -117,8 +117,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \
const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \ const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \
const unsigned long oil = get_eb_offset_in_folio(token->eb, \ const unsigned long oil = get_eb_offset_in_folio(token->eb, \
member_offset);\ member_offset);\
const int unit_size = folio_size(token->eb->folios[0]); \ const int unit_size = token->eb->folio_size; \
const int unit_shift = folio_shift(token->eb->folios[0]); \ const int unit_shift = token->eb->folio_shift; \
const int size = sizeof(u##bits); \ const int size = sizeof(u##bits); \
u8 lebytes[sizeof(u##bits)]; \ u8 lebytes[sizeof(u##bits)]; \
const int part = unit_size - oil; \ const int part = unit_size - oil; \
@ -151,7 +151,7 @@ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \
const unsigned long idx = get_eb_folio_index(eb, member_offset);\ const unsigned long idx = get_eb_folio_index(eb, member_offset);\
const unsigned long oil = get_eb_offset_in_folio(eb, \ const unsigned long oil = get_eb_offset_in_folio(eb, \
member_offset);\ member_offset);\
const int unit_size = folio_size(eb->folios[0]); \ const int unit_size = eb->folio_size; \
char *kaddr = folio_address(eb->folios[idx]); \ char *kaddr = folio_address(eb->folios[idx]); \
const int size = sizeof(u##bits); \ const int size = sizeof(u##bits); \
const int part = unit_size - oil; \ const int part = unit_size - oil; \

2
fs/btrfs/ctree.c
View File

@ -820,7 +820,7 @@ int btrfs_bin_search(struct extent_buffer *eb, int first_slot,
} }
while (low < high) { while (low < high) {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
unsigned long oil; unsigned long oil;
unsigned long offset; unsigned long offset;
struct btrfs_disk_key *tmp; struct btrfs_disk_key *tmp;

2
fs/btrfs/disk-io.c
View File

@ -193,7 +193,7 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
struct folio *folio = eb->folios[i]; struct folio *folio = eb->folios[i];
u64 start = max_t(u64, eb->start, folio_pos(folio)); u64 start = max_t(u64, eb->start, folio_pos(folio));
u64 end = min_t(u64, eb->start + eb->len, u64 end = min_t(u64, eb->start + eb->len,
folio_pos(folio) + folio_size(folio)); folio_pos(folio) + eb->folio_size);
u32 len = end - start; u32 len = end - start;
ret = btrfs_repair_io_failure(fs_info, 0, start, len, ret = btrfs_repair_io_failure(fs_info, 0, start, len,

38
fs/btrfs/extent_io.c
View File

@ -78,7 +78,7 @@ void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
eb = list_first_entry(&fs_info->allocated_ebs, eb = list_first_entry(&fs_info->allocated_ebs,
struct extent_buffer, leak_list); struct extent_buffer, leak_list);
pr_err( pr_err(
"BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n", "BTRFS: buffer leak start %llu len %u refs %d bflags %lu owner %llu\n",
eb->start, eb->len, atomic_read(&eb->refs), eb->bflags, eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
btrfs_header_owner(eb)); btrfs_header_owner(eb));
list_del(&eb->leak_list); list_del(&eb->leak_list);
@ -729,6 +729,8 @@ static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t extra_gfp)
for (int i = 0; i < num_pages; i++) for (int i = 0; i < num_pages; i++)
eb->folios[i] = page_folio(page_array[i]); eb->folios[i] = page_folio(page_array[i]);
eb->folio_size = PAGE_SIZE;
eb->folio_shift = PAGE_SHIFT;
return 0; return 0;
} }
@ -1728,10 +1730,10 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
folio_lock(folio); folio_lock(folio);
folio_clear_dirty_for_io(folio); folio_clear_dirty_for_io(folio);
folio_start_writeback(folio); folio_start_writeback(folio);
ret = bio_add_folio(&bbio->bio, folio, folio_size(folio), 0); ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
ASSERT(ret); ASSERT(ret);
wbc_account_cgroup_owner(wbc, folio_page(folio, 0), wbc_account_cgroup_owner(wbc, folio_page(folio, 0),
folio_size(folio)); eb->folio_size);
wbc->nr_to_write -= folio_nr_pages(folio); wbc->nr_to_write -= folio_nr_pages(folio);
folio_unlock(folio); folio_unlock(folio);
} }
@ -3635,7 +3637,7 @@ static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
/* For now, we should only have single-page folios for btree inode. */ /* For now, we should only have single-page folios for btree inode. */
ASSERT(folio_nr_pages(existing_folio) == 1); ASSERT(folio_nr_pages(existing_folio) == 1);
if (folio_size(existing_folio) != folio_size(eb->folios[0])) { if (folio_size(existing_folio) != eb->folio_size) {
folio_unlock(existing_folio); folio_unlock(existing_folio);
folio_put(existing_folio); folio_put(existing_folio);
return -EAGAIN; return -EAGAIN;
@ -3778,6 +3780,8 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
* and free the allocated page. * and free the allocated page.
*/ */
folio = eb->folios[i]; folio = eb->folios[i];
eb->folio_size = folio_size(folio);
eb->folio_shift = folio_shift(folio);
spin_lock(&mapping->i_private_lock); spin_lock(&mapping->i_private_lock);
/* Should not fail, as we have preallocated the memory */ /* Should not fail, as we have preallocated the memory */
ret = attach_extent_buffer_folio(eb, folio, prealloc); ret = attach_extent_buffer_folio(eb, folio, prealloc);
@ -4227,7 +4231,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
for (int i = 0; i < num_folios; i++) { for (int i = 0; i < num_folios; i++) {
struct folio *folio = eb->folios[i]; struct folio *folio = eb->folios[i];
ret = bio_add_folio(&bbio->bio, folio, folio_size(folio), 0); ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
ASSERT(ret); ASSERT(ret);
} }
} }
@ -4247,7 +4251,7 @@ static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
unsigned long len) unsigned long len)
{ {
btrfs_warn(eb->fs_info, btrfs_warn(eb->fs_info,
"access to eb bytenr %llu len %lu out of range start %lu len %lu", "access to eb bytenr %llu len %u out of range start %lu len %lu",
eb->start, eb->len, start, len); eb->start, eb->len, start, len);
WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG)); WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
@ -4276,7 +4280,7 @@ static inline int check_eb_range(const struct extent_buffer *eb,
void read_extent_buffer(const struct extent_buffer *eb, void *dstv, void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
{ {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
size_t cur; size_t cur;
size_t offset; size_t offset;
char *dst = (char *)dstv; char *dst = (char *)dstv;
@ -4316,7 +4320,7 @@ int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
void __user *dstv, void __user *dstv,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
{ {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
size_t cur; size_t cur;
size_t offset; size_t offset;
char __user *dst = (char __user *)dstv; char __user *dst = (char __user *)dstv;
@ -4356,7 +4360,7 @@ int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
{ {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
size_t cur; size_t cur;
size_t offset; size_t offset;
char *kaddr; char *kaddr;
@ -4427,7 +4431,7 @@ static void __write_extent_buffer(const struct extent_buffer *eb,
const void *srcv, unsigned long start, const void *srcv, unsigned long start,
unsigned long len, bool use_memmove) unsigned long len, bool use_memmove)
{ {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
size_t cur; size_t cur;
size_t offset; size_t offset;
char *kaddr; char *kaddr;
@ -4476,7 +4480,7 @@ void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
static void memset_extent_buffer(const struct extent_buffer *eb, int c, static void memset_extent_buffer(const struct extent_buffer *eb, int c,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
{ {
const int unit_size = folio_size(eb->folios[0]); const int unit_size = eb->folio_size;
unsigned long cur = start; unsigned long cur = start;
if (eb->addr) { if (eb->addr) {
@ -4507,7 +4511,7 @@ void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
void copy_extent_buffer_full(const struct extent_buffer *dst, void copy_extent_buffer_full(const struct extent_buffer *dst,
const struct extent_buffer *src) const struct extent_buffer *src)
{ {
const int unit_size = folio_size(src->folios[0]); const int unit_size = src->folio_size;
unsigned long cur = 0; unsigned long cur = 0;
ASSERT(dst->len == src->len); ASSERT(dst->len == src->len);
@ -4529,7 +4533,7 @@ void copy_extent_buffer(const struct extent_buffer *dst,
unsigned long dst_offset, unsigned long src_offset, unsigned long dst_offset, unsigned long src_offset,
unsigned long len) unsigned long len)
{ {
const int unit_size = folio_size(dst->folios[0]); const int unit_size = dst->folio_size;
u64 dst_len = dst->len; u64 dst_len = dst->len;
size_t cur; size_t cur;
size_t offset; size_t offset;
@ -4585,10 +4589,10 @@ static inline void eb_bitmap_offset(const struct extent_buffer *eb,
* the bitmap item in the extent buffer + the offset of the byte in the * the bitmap item in the extent buffer + the offset of the byte in the
* bitmap item. * bitmap item.
*/ */
offset = start + offset_in_folio(eb->folios[0], eb->start) + byte_offset; offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset;
*folio_index = offset >> folio_shift(eb->folios[0]); *folio_index = offset >> eb->folio_shift;
*folio_offset = offset_in_folio(eb->folios[0], offset); *folio_offset = offset_in_eb_folio(eb, offset);
} }
/* /*
@ -4702,7 +4706,7 @@ void memcpy_extent_buffer(const struct extent_buffer *dst,
unsigned long dst_offset, unsigned long src_offset, unsigned long dst_offset, unsigned long src_offset,
unsigned long len) unsigned long len)
{ {
const int unit_size = folio_size(dst->folios[0]); const int unit_size = dst->folio_size;
unsigned long cur_off = 0; unsigned long cur_off = 0;
if (check_eb_range(dst, dst_offset, len) || if (check_eb_range(dst, dst_offset, len) ||

16
fs/btrfs/extent_io.h
View File

@ -8,6 +8,7 @@
#include <linux/fiemap.h> #include <linux/fiemap.h>
#include <linux/btrfs_tree.h> #include <linux/btrfs_tree.h>
#include "compression.h" #include "compression.h"
#include "messages.h"
#include "ulist.h" #include "ulist.h"
#include "misc.h" #include "misc.h"
@ -75,7 +76,8 @@ void __cold extent_buffer_free_cachep(void);
#define INLINE_EXTENT_BUFFER_PAGES (BTRFS_MAX_METADATA_BLOCKSIZE / PAGE_SIZE) #define INLINE_EXTENT_BUFFER_PAGES (BTRFS_MAX_METADATA_BLOCKSIZE / PAGE_SIZE)
struct extent_buffer { struct extent_buffer {
u64 start; u64 start;
unsigned long len; u32 len;
u32 folio_size;
unsigned long bflags; unsigned long bflags;
struct btrfs_fs_info *fs_info; struct btrfs_fs_info *fs_info;
@ -90,6 +92,7 @@ struct extent_buffer {
int read_mirror; int read_mirror;
/* >= 0 if eb belongs to a log tree, -1 otherwise */ /* >= 0 if eb belongs to a log tree, -1 otherwise */
s8 log_index; s8 log_index;
u8 folio_shift;
struct rcu_head rcu_head; struct rcu_head rcu_head;
struct rw_semaphore lock; struct rw_semaphore lock;
@ -113,6 +116,13 @@ struct btrfs_eb_write_context {
struct btrfs_block_group *zoned_bg; struct btrfs_block_group *zoned_bg;
}; };
static inline unsigned long offset_in_eb_folio(const struct extent_buffer *eb,
u64 start)
{
ASSERT(eb->folio_size);
return start & (eb->folio_size - 1);
}
/* /*
* Get the correct offset inside the page of extent buffer. * Get the correct offset inside the page of extent buffer.
* *
@ -151,13 +161,13 @@ static inline unsigned long get_eb_folio_index(const struct extent_buffer *eb,
* the folio_shift would be large enough to always make us * the folio_shift would be large enough to always make us
* return 0 as index. * return 0 as index.
* 1.2) Several page sized folios * 1.2) Several page sized folios
* The folio_shift() would be PAGE_SHIFT, giving us the correct * The folio_shift would be PAGE_SHIFT, giving us the correct
* index. * index.
* *
* 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case
* The folio would only be page sized, and always give us 0 as index. * The folio would only be page sized, and always give us 0 as index.
*/ */
return offset >> folio_shift(eb->folios[0]); return offset >> eb->folio_shift;
} }
/* /*