btrfs: cache folio size and shift in extent_buffer

After the conversion to folio interfaces (but without the patch to
enable larger folio allocation), there is an LTP report about observable
performance drop on metadata heavy operations.

https://lore.kernel.org/linux-btrfs/202312221750.571925bd-oliver.sang@intel.com/

This drop is caused by the extra code of calculating the
folio_size()/folio_shift(), instead of the old hard coded
PAGE_SIZE/PAGE_SHIFT.

To slightly reduce the overhead, just cache both folio_size and
folio_shift in extent_buffer.

The two new members (u32 folio_size and u8 folio_shift) are stored
inside the holes of extent_buffer. folio_size is shared with len, which
is reduced to u32. The size of eb does not change.

Signed-off-by: Qu Wenruo <wqu@suse.com>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>

diff --git a/fs/btrfs/accessors.c b/fs/btrfs/accessors.c
index 1925a0919ca62f0cff4e5836ab248062c4b57922..6eb850ad37d2aef99bbc0eac57a3719be895544b 100644 (file)
--- a/fs/btrfs/accessors.c
+++ b/fs/btrfs/accessors.c
@@ -63,8 +63,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token,         \
        const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \
        const unsigned long oil = get_eb_offset_in_folio(token->eb,     \
                                                         member_offset);\
-       const int unit_size = folio_size(token->eb->folios[0]);         \
-       const int unit_shift = folio_shift(token->eb->folios[0]);       \
+       const int unit_size = token->eb->folio_size;                    \
+       const int unit_shift = token->eb->folio_shift;                  \
        const int size = sizeof(u##bits);                               \
        u8 lebytes[sizeof(u##bits)];                                    \
        const int part = unit_size - oil;                               \
@@ -94,7 +94,7 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb,              \
        const unsigned long idx = get_eb_folio_index(eb, member_offset);\
        const unsigned long oil = get_eb_offset_in_folio(eb,            \
                                                         member_offset);\
-       const int unit_size = folio_size(eb->folios[0]);                \
+       const int unit_size = eb->folio_size;                           \
        char *kaddr = folio_address(eb->folios[idx]);                   \
        const int size = sizeof(u##bits);                               \
        const int part = unit_size - oil;                               \
@@ -117,8 +117,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token,          \
        const unsigned long idx = get_eb_folio_index(token->eb, member_offset); \
        const unsigned long oil = get_eb_offset_in_folio(token->eb,     \
                                                         member_offset);\
-       const int unit_size = folio_size(token->eb->folios[0]);         \
-       const int unit_shift = folio_shift(token->eb->folios[0]);       \
+       const int unit_size = token->eb->folio_size;                    \
+       const int unit_shift = token->eb->folio_shift;                  \
        const int size = sizeof(u##bits);                               \
        u8 lebytes[sizeof(u##bits)];                                    \
        const int part = unit_size - oil;                               \
@@ -151,7 +151,7 @@ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr,    \
        const unsigned long idx = get_eb_folio_index(eb, member_offset);\
        const unsigned long oil = get_eb_offset_in_folio(eb,            \
                                                         member_offset);\
-       const int unit_size = folio_size(eb->folios[0]);                \
+       const int unit_size = eb->folio_size;                           \
        char *kaddr = folio_address(eb->folios[idx]);                   \
        const int size = sizeof(u##bits);                               \
        const int part = unit_size - oil;                               \

diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index e65e012bac5531eca567ceb31ce5aea5b30c4435..33145da449cc8db3b248d3ec5ea7bba878277992 100644 (file)
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -820,7 +820,7 @@ int btrfs_bin_search(struct extent_buffer *eb, int first_slot,
        }
 
        while (low < high) {
-               const int unit_size = folio_size(eb->folios[0]);
+               const int unit_size = eb->folio_size;
                unsigned long oil;
                unsigned long offset;
                struct btrfs_disk_key *tmp;

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index c843563914cad08e2dd84ef1741e19d933f092ee..f405cc27886124fb47d2678a7e2138de594dbd3a 100644 (file)
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -193,7 +193,7 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
                struct folio *folio = eb->folios[i];
                u64 start = max_t(u64, eb->start, folio_pos(folio));
                u64 end = min_t(u64, eb->start + eb->len,
-                               folio_pos(folio) + folio_size(folio));
+                               folio_pos(folio) + eb->folio_size);
                u32 len = end - start;
 
                ret = btrfs_repair_io_failure(fs_info, 0, start, len,

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index adc74abe4bfac0ad1678206cd007998454868d6b..a8c8b9d8c40091a07230a22881b27d0c6c7bbadb 100644 (file)
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -78,7 +78,7 @@ void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
                eb = list_first_entry(&fs_info->allocated_ebs,
                                      struct extent_buffer, leak_list);
                pr_err(
-       "BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
+       "BTRFS: buffer leak start %llu len %u refs %d bflags %lu owner %llu\n",
                       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
                       btrfs_header_owner(eb));
                list_del(&eb->leak_list);
@@ -729,6 +729,8 @@ static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t extra_gfp)
 
        for (int i = 0; i < num_pages; i++)
                eb->folios[i] = page_folio(page_array[i]);
+       eb->folio_size = PAGE_SIZE;
+       eb->folio_shift = PAGE_SHIFT;
        return 0;
 }
 
@@ -1728,10 +1730,10 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
                        folio_lock(folio);
                        folio_clear_dirty_for_io(folio);
                        folio_start_writeback(folio);
-                       ret = bio_add_folio(&bbio->bio, folio, folio_size(folio), 0);
+                       ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
                        ASSERT(ret);
                        wbc_account_cgroup_owner(wbc, folio_page(folio, 0),
-                                                folio_size(folio));
+                                                eb->folio_size);
                        wbc->nr_to_write -= folio_nr_pages(folio);
                        folio_unlock(folio);
                }
@@ -3635,7 +3637,7 @@ retry:
        /* For now, we should only have single-page folios for btree inode. */
        ASSERT(folio_nr_pages(existing_folio) == 1);
 
-       if (folio_size(existing_folio) != folio_size(eb->folios[0])) {
+       if (folio_size(existing_folio) != eb->folio_size) {
                folio_unlock(existing_folio);
                folio_put(existing_folio);
                return -EAGAIN;
@@ -3778,6 +3780,8 @@ reallocate:
                 * and free the allocated page.
                 */
                folio = eb->folios[i];
+               eb->folio_size = folio_size(folio);
+               eb->folio_shift = folio_shift(folio);
                spin_lock(&mapping->i_private_lock);
                /* Should not fail, as we have preallocated the memory */
                ret = attach_extent_buffer_folio(eb, folio, prealloc);
@@ -4227,7 +4231,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
                for (int i = 0; i < num_folios; i++) {
                        struct folio *folio = eb->folios[i];
 
-                       ret = bio_add_folio(&bbio->bio, folio, folio_size(folio), 0);
+                       ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
                        ASSERT(ret);
                }
        }
@@ -4247,7 +4251,7 @@ static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
                            unsigned long len)
 {
        btrfs_warn(eb->fs_info,
-               "access to eb bytenr %llu len %lu out of range start %lu len %lu",
+               "access to eb bytenr %llu len %u out of range start %lu len %lu",
                eb->start, eb->len, start, len);
        WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 
@@ -4276,7 +4280,7 @@ static inline int check_eb_range(const struct extent_buffer *eb,
 void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
                        unsigned long start, unsigned long len)
 {
-       const int unit_size = folio_size(eb->folios[0]);
+       const int unit_size = eb->folio_size;
        size_t cur;
        size_t offset;
        char *dst = (char *)dstv;
@@ -4316,7 +4320,7 @@ int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
                                       void __user *dstv,
                                       unsigned long start, unsigned long len)
 {
-       const int unit_size = folio_size(eb->folios[0]);
+       const int unit_size = eb->folio_size;
        size_t cur;
        size_t offset;
        char __user *dst = (char __user *)dstv;
@@ -4356,7 +4360,7 @@ int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
                         unsigned long start, unsigned long len)
 {
-       const int unit_size = folio_size(eb->folios[0]);
+       const int unit_size = eb->folio_size;
        size_t cur;
        size_t offset;
        char *kaddr;
@@ -4427,7 +4431,7 @@ static void __write_extent_buffer(const struct extent_buffer *eb,
                                  const void *srcv, unsigned long start,
                                  unsigned long len, bool use_memmove)
 {
-       const int unit_size = folio_size(eb->folios[0]);
+       const int unit_size = eb->folio_size;
        size_t cur;
        size_t offset;
        char *kaddr;
@@ -4476,7 +4480,7 @@ void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
 static void memset_extent_buffer(const struct extent_buffer *eb, int c,
                                 unsigned long start, unsigned long len)
 {
-       const int unit_size = folio_size(eb->folios[0]);
+       const int unit_size = eb->folio_size;
        unsigned long cur = start;
 
        if (eb->addr) {
@@ -4507,7 +4511,7 @@ void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
 void copy_extent_buffer_full(const struct extent_buffer *dst,
                             const struct extent_buffer *src)
 {
-       const int unit_size = folio_size(src->folios[0]);
+       const int unit_size = src->folio_size;
        unsigned long cur = 0;
 
        ASSERT(dst->len == src->len);
@@ -4529,7 +4533,7 @@ void copy_extent_buffer(const struct extent_buffer *dst,
                        unsigned long dst_offset, unsigned long src_offset,
                        unsigned long len)
 {
-       const int unit_size = folio_size(dst->folios[0]);
+       const int unit_size = dst->folio_size;
        u64 dst_len = dst->len;
        size_t cur;
        size_t offset;
@@ -4585,10 +4589,10 @@ static inline void eb_bitmap_offset(const struct extent_buffer *eb,
         * the bitmap item in the extent buffer + the offset of the byte in the
         * bitmap item.
         */
-       offset = start + offset_in_folio(eb->folios[0], eb->start) + byte_offset;
+       offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset;
 
-       *folio_index = offset >> folio_shift(eb->folios[0]);
-       *folio_offset = offset_in_folio(eb->folios[0], offset);
+       *folio_index = offset >> eb->folio_shift;
+       *folio_offset = offset_in_eb_folio(eb, offset);
 }
 
 /*
@@ -4702,7 +4706,7 @@ void memcpy_extent_buffer(const struct extent_buffer *dst,
                          unsigned long dst_offset, unsigned long src_offset,
                          unsigned long len)
 {
-       const int unit_size = folio_size(dst->folios[0]);
+       const int unit_size = dst->folio_size;
        unsigned long cur_off = 0;
 
        if (check_eb_range(dst, dst_offset, len) ||

diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index 46050500529bff3720c3f117ca593646abaadc65..8e5639597800a7d92b6030fa66a2f607ede039ac 100644 (file)
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -8,6 +8,7 @@
 #include <linux/fiemap.h>
 #include <linux/btrfs_tree.h>
 #include "compression.h"
+#include "messages.h"
 #include "ulist.h"
 #include "misc.h"
 
@@ -75,7 +76,8 @@ void __cold extent_buffer_free_cachep(void);
 #define INLINE_EXTENT_BUFFER_PAGES     (BTRFS_MAX_METADATA_BLOCKSIZE / PAGE_SIZE)
 struct extent_buffer {
        u64 start;
-       unsigned long len;
+       u32 len;
+       u32 folio_size;
        unsigned long bflags;
        struct btrfs_fs_info *fs_info;
 
@@ -90,6 +92,7 @@ struct extent_buffer {
        int read_mirror;
        /* >= 0 if eb belongs to a log tree, -1 otherwise */
        s8 log_index;
+       u8 folio_shift;
        struct rcu_head rcu_head;
 
        struct rw_semaphore lock;
@@ -113,6 +116,13 @@ struct btrfs_eb_write_context {
        struct btrfs_block_group *zoned_bg;
 };
 
+static inline unsigned long offset_in_eb_folio(const struct extent_buffer *eb,
+                                              u64 start)
+{
+       ASSERT(eb->folio_size);
+       return start & (eb->folio_size - 1);
+}
+
 /*
  * Get the correct offset inside the page of extent buffer.
  *
@@ -151,13 +161,13 @@ static inline unsigned long get_eb_folio_index(const struct extent_buffer *eb,
         *         the folio_shift would be large enough to always make us
         *         return 0 as index.
         *    1.2) Several page sized folios
-        *         The folio_shift() would be PAGE_SHIFT, giving us the correct
+        *         The folio_shift would be PAGE_SHIFT, giving us the correct
         *         index.
         *
         * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case
         *    The folio would only be page sized, and always give us 0 as index.
         */
-       return offset >> folio_shift(eb->folios[0]);
+       return offset >> eb->folio_shift;
 }
 
 /*