1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/prefetch.h>
15 #include <linux/cleancache.h>
16 #include "extent_io.h"
17 #include "extent_map.h"
19 #include "btrfs_inode.h"
21 #include "check-integrity.h"
23 #include "rcu-string.h"
27 static struct kmem_cache *extent_state_cache;
28 static struct kmem_cache *extent_buffer_cache;
29 static struct bio_set btrfs_bioset;
31 static inline bool extent_state_in_tree(const struct extent_state *state)
33 return !RB_EMPTY_NODE(&state->rb_node);
36 #ifdef CONFIG_BTRFS_DEBUG
37 static LIST_HEAD(buffers);
38 static LIST_HEAD(states);
40 static DEFINE_SPINLOCK(leak_lock);
43 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
47 spin_lock_irqsave(&leak_lock, flags);
49 spin_unlock_irqrestore(&leak_lock, flags);
53 void btrfs_leak_debug_del(struct list_head *entry)
57 spin_lock_irqsave(&leak_lock, flags);
59 spin_unlock_irqrestore(&leak_lock, flags);
63 void btrfs_leak_debug_check(void)
65 struct extent_state *state;
66 struct extent_buffer *eb;
68 while (!list_empty(&states)) {
69 state = list_entry(states.next, struct extent_state, leak_list);
70 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
71 state->start, state->end, state->state,
72 extent_state_in_tree(state),
73 refcount_read(&state->refs));
74 list_del(&state->leak_list);
75 kmem_cache_free(extent_state_cache, state);
78 while (!list_empty(&buffers)) {
79 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
80 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
82 list_del(&eb->leak_list);
83 kmem_cache_free(extent_buffer_cache, eb);
87 #define btrfs_debug_check_extent_io_range(tree, start, end) \
88 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
89 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
90 struct extent_io_tree *tree, u64 start, u64 end)
92 struct inode *inode = tree->private_data;
95 if (!inode || !is_data_inode(inode))
98 isize = i_size_read(inode);
99 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
100 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
101 "%s: ino %llu isize %llu odd range [%llu,%llu]",
102 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
106 #define btrfs_leak_debug_add(new, head) do {} while (0)
107 #define btrfs_leak_debug_del(entry) do {} while (0)
108 #define btrfs_leak_debug_check() do {} while (0)
109 #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
112 #define BUFFER_LRU_MAX 64
117 struct rb_node rb_node;
120 struct extent_page_data {
122 struct extent_io_tree *tree;
123 /* tells writepage not to lock the state bits for this range
124 * it still does the unlocking
126 unsigned int extent_locked:1;
128 /* tells the submit_bio code to use REQ_SYNC */
129 unsigned int sync_io:1;
132 static int add_extent_changeset(struct extent_state *state, unsigned bits,
133 struct extent_changeset *changeset,
140 if (set && (state->state & bits) == bits)
142 if (!set && (state->state & bits) == 0)
144 changeset->bytes_changed += state->end - state->start + 1;
145 ret = ulist_add(&changeset->range_changed, state->start, state->end,
150 static void flush_write_bio(struct extent_page_data *epd);
152 int __init extent_io_init(void)
154 extent_state_cache = kmem_cache_create("btrfs_extent_state",
155 sizeof(struct extent_state), 0,
156 SLAB_MEM_SPREAD, NULL);
157 if (!extent_state_cache)
160 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
161 sizeof(struct extent_buffer), 0,
162 SLAB_MEM_SPREAD, NULL);
163 if (!extent_buffer_cache)
164 goto free_state_cache;
166 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
167 offsetof(struct btrfs_io_bio, bio),
169 goto free_buffer_cache;
171 if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
177 bioset_exit(&btrfs_bioset);
180 kmem_cache_destroy(extent_buffer_cache);
181 extent_buffer_cache = NULL;
184 kmem_cache_destroy(extent_state_cache);
185 extent_state_cache = NULL;
189 void __cold extent_io_exit(void)
191 btrfs_leak_debug_check();
194 * Make sure all delayed rcu free are flushed before we
198 kmem_cache_destroy(extent_state_cache);
199 kmem_cache_destroy(extent_buffer_cache);
200 bioset_exit(&btrfs_bioset);
203 void extent_io_tree_init(struct extent_io_tree *tree,
206 tree->state = RB_ROOT;
208 tree->dirty_bytes = 0;
209 spin_lock_init(&tree->lock);
210 tree->private_data = private_data;
213 static struct extent_state *alloc_extent_state(gfp_t mask)
215 struct extent_state *state;
218 * The given mask might be not appropriate for the slab allocator,
219 * drop the unsupported bits
221 mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
222 state = kmem_cache_alloc(extent_state_cache, mask);
226 state->failrec = NULL;
227 RB_CLEAR_NODE(&state->rb_node);
228 btrfs_leak_debug_add(&state->leak_list, &states);
229 refcount_set(&state->refs, 1);
230 init_waitqueue_head(&state->wq);
231 trace_alloc_extent_state(state, mask, _RET_IP_);
235 void free_extent_state(struct extent_state *state)
239 if (refcount_dec_and_test(&state->refs)) {
240 WARN_ON(extent_state_in_tree(state));
241 btrfs_leak_debug_del(&state->leak_list);
242 trace_free_extent_state(state, _RET_IP_);
243 kmem_cache_free(extent_state_cache, state);
247 static struct rb_node *tree_insert(struct rb_root *root,
248 struct rb_node *search_start,
250 struct rb_node *node,
251 struct rb_node ***p_in,
252 struct rb_node **parent_in)
255 struct rb_node *parent = NULL;
256 struct tree_entry *entry;
258 if (p_in && parent_in) {
264 p = search_start ? &search_start : &root->rb_node;
267 entry = rb_entry(parent, struct tree_entry, rb_node);
269 if (offset < entry->start)
271 else if (offset > entry->end)
278 rb_link_node(node, parent, p);
279 rb_insert_color(node, root);
283 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
284 struct rb_node **prev_ret,
285 struct rb_node **next_ret,
286 struct rb_node ***p_ret,
287 struct rb_node **parent_ret)
289 struct rb_root *root = &tree->state;
290 struct rb_node **n = &root->rb_node;
291 struct rb_node *prev = NULL;
292 struct rb_node *orig_prev = NULL;
293 struct tree_entry *entry;
294 struct tree_entry *prev_entry = NULL;
298 entry = rb_entry(prev, struct tree_entry, rb_node);
301 if (offset < entry->start)
303 else if (offset > entry->end)
316 while (prev && offset > prev_entry->end) {
317 prev = rb_next(prev);
318 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
325 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
326 while (prev && offset < prev_entry->start) {
327 prev = rb_prev(prev);
328 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
335 static inline struct rb_node *
336 tree_search_for_insert(struct extent_io_tree *tree,
338 struct rb_node ***p_ret,
339 struct rb_node **parent_ret)
341 struct rb_node *prev = NULL;
344 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
350 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
353 return tree_search_for_insert(tree, offset, NULL, NULL);
357 * utility function to look for merge candidates inside a given range.
358 * Any extents with matching state are merged together into a single
359 * extent in the tree. Extents with EXTENT_IO in their state field
360 * are not merged because the end_io handlers need to be able to do
361 * operations on them without sleeping (or doing allocations/splits).
363 * This should be called with the tree lock held.
365 static void merge_state(struct extent_io_tree *tree,
366 struct extent_state *state)
368 struct extent_state *other;
369 struct rb_node *other_node;
371 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
374 other_node = rb_prev(&state->rb_node);
376 other = rb_entry(other_node, struct extent_state, rb_node);
377 if (other->end == state->start - 1 &&
378 other->state == state->state) {
379 if (tree->private_data &&
380 is_data_inode(tree->private_data))
381 btrfs_merge_delalloc_extent(tree->private_data,
383 state->start = other->start;
384 rb_erase(&other->rb_node, &tree->state);
385 RB_CLEAR_NODE(&other->rb_node);
386 free_extent_state(other);
389 other_node = rb_next(&state->rb_node);
391 other = rb_entry(other_node, struct extent_state, rb_node);
392 if (other->start == state->end + 1 &&
393 other->state == state->state) {
394 if (tree->private_data &&
395 is_data_inode(tree->private_data))
396 btrfs_merge_delalloc_extent(tree->private_data,
398 state->end = other->end;
399 rb_erase(&other->rb_node, &tree->state);
400 RB_CLEAR_NODE(&other->rb_node);
401 free_extent_state(other);
406 static void set_state_bits(struct extent_io_tree *tree,
407 struct extent_state *state, unsigned *bits,
408 struct extent_changeset *changeset);
411 * insert an extent_state struct into the tree. 'bits' are set on the
412 * struct before it is inserted.
414 * This may return -EEXIST if the extent is already there, in which case the
415 * state struct is freed.
417 * The tree lock is not taken internally. This is a utility function and
418 * probably isn't what you want to call (see set/clear_extent_bit).
420 static int insert_state(struct extent_io_tree *tree,
421 struct extent_state *state, u64 start, u64 end,
423 struct rb_node **parent,
424 unsigned *bits, struct extent_changeset *changeset)
426 struct rb_node *node;
429 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
431 state->start = start;
434 set_state_bits(tree, state, bits, changeset);
436 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
438 struct extent_state *found;
439 found = rb_entry(node, struct extent_state, rb_node);
440 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
441 found->start, found->end, start, end);
444 merge_state(tree, state);
449 * split a given extent state struct in two, inserting the preallocated
450 * struct 'prealloc' as the newly created second half. 'split' indicates an
451 * offset inside 'orig' where it should be split.
454 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
455 * are two extent state structs in the tree:
456 * prealloc: [orig->start, split - 1]
457 * orig: [ split, orig->end ]
459 * The tree locks are not taken by this function. They need to be held
462 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
463 struct extent_state *prealloc, u64 split)
465 struct rb_node *node;
467 if (tree->private_data && is_data_inode(tree->private_data))
468 btrfs_split_delalloc_extent(tree->private_data, orig, split);
470 prealloc->start = orig->start;
471 prealloc->end = split - 1;
472 prealloc->state = orig->state;
475 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
476 &prealloc->rb_node, NULL, NULL);
478 free_extent_state(prealloc);
484 static struct extent_state *next_state(struct extent_state *state)
486 struct rb_node *next = rb_next(&state->rb_node);
488 return rb_entry(next, struct extent_state, rb_node);
494 * utility function to clear some bits in an extent state struct.
495 * it will optionally wake up anyone waiting on this state (wake == 1).
497 * If no bits are set on the state struct after clearing things, the
498 * struct is freed and removed from the tree
500 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
501 struct extent_state *state,
502 unsigned *bits, int wake,
503 struct extent_changeset *changeset)
505 struct extent_state *next;
506 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
509 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
510 u64 range = state->end - state->start + 1;
511 WARN_ON(range > tree->dirty_bytes);
512 tree->dirty_bytes -= range;
515 if (tree->private_data && is_data_inode(tree->private_data))
516 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
518 ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
520 state->state &= ~bits_to_clear;
523 if (state->state == 0) {
524 next = next_state(state);
525 if (extent_state_in_tree(state)) {
526 rb_erase(&state->rb_node, &tree->state);
527 RB_CLEAR_NODE(&state->rb_node);
528 free_extent_state(state);
533 merge_state(tree, state);
534 next = next_state(state);
539 static struct extent_state *
540 alloc_extent_state_atomic(struct extent_state *prealloc)
543 prealloc = alloc_extent_state(GFP_ATOMIC);
548 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
550 struct inode *inode = tree->private_data;
552 btrfs_panic(btrfs_sb(inode->i_sb), err,
553 "locking error: extent tree was modified by another thread while locked");
557 * clear some bits on a range in the tree. This may require splitting
558 * or inserting elements in the tree, so the gfp mask is used to
559 * indicate which allocations or sleeping are allowed.
561 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
562 * the given range from the tree regardless of state (ie for truncate).
564 * the range [start, end] is inclusive.
566 * This takes the tree lock, and returns 0 on success and < 0 on error.
568 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
569 unsigned bits, int wake, int delete,
570 struct extent_state **cached_state,
571 gfp_t mask, struct extent_changeset *changeset)
573 struct extent_state *state;
574 struct extent_state *cached;
575 struct extent_state *prealloc = NULL;
576 struct rb_node *node;
581 btrfs_debug_check_extent_io_range(tree, start, end);
583 if (bits & EXTENT_DELALLOC)
584 bits |= EXTENT_NORESERVE;
587 bits |= ~EXTENT_CTLBITS;
589 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
592 if (!prealloc && gfpflags_allow_blocking(mask)) {
594 * Don't care for allocation failure here because we might end
595 * up not needing the pre-allocated extent state at all, which
596 * is the case if we only have in the tree extent states that
597 * cover our input range and don't cover too any other range.
598 * If we end up needing a new extent state we allocate it later.
600 prealloc = alloc_extent_state(mask);
603 spin_lock(&tree->lock);
605 cached = *cached_state;
608 *cached_state = NULL;
612 if (cached && extent_state_in_tree(cached) &&
613 cached->start <= start && cached->end > start) {
615 refcount_dec(&cached->refs);
620 free_extent_state(cached);
623 * this search will find the extents that end after
626 node = tree_search(tree, start);
629 state = rb_entry(node, struct extent_state, rb_node);
631 if (state->start > end)
633 WARN_ON(state->end < start);
634 last_end = state->end;
636 /* the state doesn't have the wanted bits, go ahead */
637 if (!(state->state & bits)) {
638 state = next_state(state);
643 * | ---- desired range ---- |
645 * | ------------- state -------------- |
647 * We need to split the extent we found, and may flip
648 * bits on second half.
650 * If the extent we found extends past our range, we
651 * just split and search again. It'll get split again
652 * the next time though.
654 * If the extent we found is inside our range, we clear
655 * the desired bit on it.
658 if (state->start < start) {
659 prealloc = alloc_extent_state_atomic(prealloc);
661 err = split_state(tree, state, prealloc, start);
663 extent_io_tree_panic(tree, err);
668 if (state->end <= end) {
669 state = clear_state_bit(tree, state, &bits, wake,
676 * | ---- desired range ---- |
678 * We need to split the extent, and clear the bit
681 if (state->start <= end && state->end > end) {
682 prealloc = alloc_extent_state_atomic(prealloc);
684 err = split_state(tree, state, prealloc, end + 1);
686 extent_io_tree_panic(tree, err);
691 clear_state_bit(tree, prealloc, &bits, wake, changeset);
697 state = clear_state_bit(tree, state, &bits, wake, changeset);
699 if (last_end == (u64)-1)
701 start = last_end + 1;
702 if (start <= end && state && !need_resched())
708 spin_unlock(&tree->lock);
709 if (gfpflags_allow_blocking(mask))
714 spin_unlock(&tree->lock);
716 free_extent_state(prealloc);
722 static void wait_on_state(struct extent_io_tree *tree,
723 struct extent_state *state)
724 __releases(tree->lock)
725 __acquires(tree->lock)
728 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
729 spin_unlock(&tree->lock);
731 spin_lock(&tree->lock);
732 finish_wait(&state->wq, &wait);
736 * waits for one or more bits to clear on a range in the state tree.
737 * The range [start, end] is inclusive.
738 * The tree lock is taken by this function
740 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
743 struct extent_state *state;
744 struct rb_node *node;
746 btrfs_debug_check_extent_io_range(tree, start, end);
748 spin_lock(&tree->lock);
752 * this search will find all the extents that end after
755 node = tree_search(tree, start);
760 state = rb_entry(node, struct extent_state, rb_node);
762 if (state->start > end)
765 if (state->state & bits) {
766 start = state->start;
767 refcount_inc(&state->refs);
768 wait_on_state(tree, state);
769 free_extent_state(state);
772 start = state->end + 1;
777 if (!cond_resched_lock(&tree->lock)) {
778 node = rb_next(node);
783 spin_unlock(&tree->lock);
786 static void set_state_bits(struct extent_io_tree *tree,
787 struct extent_state *state,
788 unsigned *bits, struct extent_changeset *changeset)
790 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
793 if (tree->private_data && is_data_inode(tree->private_data))
794 btrfs_set_delalloc_extent(tree->private_data, state, bits);
796 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
797 u64 range = state->end - state->start + 1;
798 tree->dirty_bytes += range;
800 ret = add_extent_changeset(state, bits_to_set, changeset, 1);
802 state->state |= bits_to_set;
805 static void cache_state_if_flags(struct extent_state *state,
806 struct extent_state **cached_ptr,
809 if (cached_ptr && !(*cached_ptr)) {
810 if (!flags || (state->state & flags)) {
812 refcount_inc(&state->refs);
817 static void cache_state(struct extent_state *state,
818 struct extent_state **cached_ptr)
820 return cache_state_if_flags(state, cached_ptr,
821 EXTENT_IOBITS | EXTENT_BOUNDARY);
825 * set some bits on a range in the tree. This may require allocations or
826 * sleeping, so the gfp mask is used to indicate what is allowed.
828 * If any of the exclusive bits are set, this will fail with -EEXIST if some
829 * part of the range already has the desired bits set. The start of the
830 * existing range is returned in failed_start in this case.
832 * [start, end] is inclusive This takes the tree lock.
835 static int __must_check
836 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
837 unsigned bits, unsigned exclusive_bits,
838 u64 *failed_start, struct extent_state **cached_state,
839 gfp_t mask, struct extent_changeset *changeset)
841 struct extent_state *state;
842 struct extent_state *prealloc = NULL;
843 struct rb_node *node;
845 struct rb_node *parent;
850 btrfs_debug_check_extent_io_range(tree, start, end);
853 if (!prealloc && gfpflags_allow_blocking(mask)) {
855 * Don't care for allocation failure here because we might end
856 * up not needing the pre-allocated extent state at all, which
857 * is the case if we only have in the tree extent states that
858 * cover our input range and don't cover too any other range.
859 * If we end up needing a new extent state we allocate it later.
861 prealloc = alloc_extent_state(mask);
864 spin_lock(&tree->lock);
865 if (cached_state && *cached_state) {
866 state = *cached_state;
867 if (state->start <= start && state->end > start &&
868 extent_state_in_tree(state)) {
869 node = &state->rb_node;
874 * this search will find all the extents that end after
877 node = tree_search_for_insert(tree, start, &p, &parent);
879 prealloc = alloc_extent_state_atomic(prealloc);
881 err = insert_state(tree, prealloc, start, end,
882 &p, &parent, &bits, changeset);
884 extent_io_tree_panic(tree, err);
886 cache_state(prealloc, cached_state);
890 state = rb_entry(node, struct extent_state, rb_node);
892 last_start = state->start;
893 last_end = state->end;
896 * | ---- desired range ---- |
899 * Just lock what we found and keep going
901 if (state->start == start && state->end <= end) {
902 if (state->state & exclusive_bits) {
903 *failed_start = state->start;
908 set_state_bits(tree, state, &bits, changeset);
909 cache_state(state, cached_state);
910 merge_state(tree, state);
911 if (last_end == (u64)-1)
913 start = last_end + 1;
914 state = next_state(state);
915 if (start < end && state && state->start == start &&
922 * | ---- desired range ---- |
925 * | ------------- state -------------- |
927 * We need to split the extent we found, and may flip bits on
930 * If the extent we found extends past our
931 * range, we just split and search again. It'll get split
932 * again the next time though.
934 * If the extent we found is inside our range, we set the
937 if (state->start < start) {
938 if (state->state & exclusive_bits) {
939 *failed_start = start;
944 prealloc = alloc_extent_state_atomic(prealloc);
946 err = split_state(tree, state, prealloc, start);
948 extent_io_tree_panic(tree, err);
953 if (state->end <= end) {
954 set_state_bits(tree, state, &bits, changeset);
955 cache_state(state, cached_state);
956 merge_state(tree, state);
957 if (last_end == (u64)-1)
959 start = last_end + 1;
960 state = next_state(state);
961 if (start < end && state && state->start == start &&
968 * | ---- desired range ---- |
969 * | state | or | state |
971 * There's a hole, we need to insert something in it and
972 * ignore the extent we found.
974 if (state->start > start) {
976 if (end < last_start)
979 this_end = last_start - 1;
981 prealloc = alloc_extent_state_atomic(prealloc);
985 * Avoid to free 'prealloc' if it can be merged with
988 err = insert_state(tree, prealloc, start, this_end,
989 NULL, NULL, &bits, changeset);
991 extent_io_tree_panic(tree, err);
993 cache_state(prealloc, cached_state);
995 start = this_end + 1;
999 * | ---- desired range ---- |
1001 * We need to split the extent, and set the bit
1004 if (state->start <= end && state->end > end) {
1005 if (state->state & exclusive_bits) {
1006 *failed_start = start;
1011 prealloc = alloc_extent_state_atomic(prealloc);
1013 err = split_state(tree, state, prealloc, end + 1);
1015 extent_io_tree_panic(tree, err);
1017 set_state_bits(tree, prealloc, &bits, changeset);
1018 cache_state(prealloc, cached_state);
1019 merge_state(tree, prealloc);
1027 spin_unlock(&tree->lock);
1028 if (gfpflags_allow_blocking(mask))
1033 spin_unlock(&tree->lock);
1035 free_extent_state(prealloc);
1041 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1042 unsigned bits, u64 * failed_start,
1043 struct extent_state **cached_state, gfp_t mask)
1045 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1046 cached_state, mask, NULL);
1051 * convert_extent_bit - convert all bits in a given range from one bit to
1053 * @tree: the io tree to search
1054 * @start: the start offset in bytes
1055 * @end: the end offset in bytes (inclusive)
1056 * @bits: the bits to set in this range
1057 * @clear_bits: the bits to clear in this range
1058 * @cached_state: state that we're going to cache
1060 * This will go through and set bits for the given range. If any states exist
1061 * already in this range they are set with the given bit and cleared of the
1062 * clear_bits. This is only meant to be used by things that are mergeable, ie
1063 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1064 * boundary bits like LOCK.
1066 * All allocations are done with GFP_NOFS.
1068 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1069 unsigned bits, unsigned clear_bits,
1070 struct extent_state **cached_state)
1072 struct extent_state *state;
1073 struct extent_state *prealloc = NULL;
1074 struct rb_node *node;
1076 struct rb_node *parent;
1080 bool first_iteration = true;
1082 btrfs_debug_check_extent_io_range(tree, start, end);
1087 * Best effort, don't worry if extent state allocation fails
1088 * here for the first iteration. We might have a cached state
1089 * that matches exactly the target range, in which case no
1090 * extent state allocations are needed. We'll only know this
1091 * after locking the tree.
1093 prealloc = alloc_extent_state(GFP_NOFS);
1094 if (!prealloc && !first_iteration)
1098 spin_lock(&tree->lock);
1099 if (cached_state && *cached_state) {
1100 state = *cached_state;
1101 if (state->start <= start && state->end > start &&
1102 extent_state_in_tree(state)) {
1103 node = &state->rb_node;
1109 * this search will find all the extents that end after
1112 node = tree_search_for_insert(tree, start, &p, &parent);
1114 prealloc = alloc_extent_state_atomic(prealloc);
1119 err = insert_state(tree, prealloc, start, end,
1120 &p, &parent, &bits, NULL);
1122 extent_io_tree_panic(tree, err);
1123 cache_state(prealloc, cached_state);
1127 state = rb_entry(node, struct extent_state, rb_node);
1129 last_start = state->start;
1130 last_end = state->end;
1133 * | ---- desired range ---- |
1136 * Just lock what we found and keep going
1138 if (state->start == start && state->end <= end) {
1139 set_state_bits(tree, state, &bits, NULL);
1140 cache_state(state, cached_state);
1141 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1142 if (last_end == (u64)-1)
1144 start = last_end + 1;
1145 if (start < end && state && state->start == start &&
1152 * | ---- desired range ---- |
1155 * | ------------- state -------------- |
1157 * We need to split the extent we found, and may flip bits on
1160 * If the extent we found extends past our
1161 * range, we just split and search again. It'll get split
1162 * again the next time though.
1164 * If the extent we found is inside our range, we set the
1165 * desired bit on it.
1167 if (state->start < start) {
1168 prealloc = alloc_extent_state_atomic(prealloc);
1173 err = split_state(tree, state, prealloc, start);
1175 extent_io_tree_panic(tree, err);
1179 if (state->end <= end) {
1180 set_state_bits(tree, state, &bits, NULL);
1181 cache_state(state, cached_state);
1182 state = clear_state_bit(tree, state, &clear_bits, 0,
1184 if (last_end == (u64)-1)
1186 start = last_end + 1;
1187 if (start < end && state && state->start == start &&
1194 * | ---- desired range ---- |
1195 * | state | or | state |
1197 * There's a hole, we need to insert something in it and
1198 * ignore the extent we found.
1200 if (state->start > start) {
1202 if (end < last_start)
1205 this_end = last_start - 1;
1207 prealloc = alloc_extent_state_atomic(prealloc);
1214 * Avoid to free 'prealloc' if it can be merged with
1217 err = insert_state(tree, prealloc, start, this_end,
1218 NULL, NULL, &bits, NULL);
1220 extent_io_tree_panic(tree, err);
1221 cache_state(prealloc, cached_state);
1223 start = this_end + 1;
1227 * | ---- desired range ---- |
1229 * We need to split the extent, and set the bit
1232 if (state->start <= end && state->end > end) {
1233 prealloc = alloc_extent_state_atomic(prealloc);
1239 err = split_state(tree, state, prealloc, end + 1);
1241 extent_io_tree_panic(tree, err);
1243 set_state_bits(tree, prealloc, &bits, NULL);
1244 cache_state(prealloc, cached_state);
1245 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1253 spin_unlock(&tree->lock);
1255 first_iteration = false;
1259 spin_unlock(&tree->lock);
1261 free_extent_state(prealloc);
1266 /* wrappers around set/clear extent bit */
1267 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1268 unsigned bits, struct extent_changeset *changeset)
1271 * We don't support EXTENT_LOCKED yet, as current changeset will
1272 * record any bits changed, so for EXTENT_LOCKED case, it will
1273 * either fail with -EEXIST or changeset will record the whole
1276 BUG_ON(bits & EXTENT_LOCKED);
1278 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1282 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1283 unsigned bits, int wake, int delete,
1284 struct extent_state **cached)
1286 return __clear_extent_bit(tree, start, end, bits, wake, delete,
1287 cached, GFP_NOFS, NULL);
1290 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1291 unsigned bits, struct extent_changeset *changeset)
1294 * Don't support EXTENT_LOCKED case, same reason as
1295 * set_record_extent_bits().
1297 BUG_ON(bits & EXTENT_LOCKED);
1299 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1304 * either insert or lock state struct between start and end use mask to tell
1305 * us if waiting is desired.
1307 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1308 struct extent_state **cached_state)
1314 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1315 EXTENT_LOCKED, &failed_start,
1316 cached_state, GFP_NOFS, NULL);
1317 if (err == -EEXIST) {
1318 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1319 start = failed_start;
1322 WARN_ON(start > end);
1327 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1332 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1333 &failed_start, NULL, GFP_NOFS, NULL);
1334 if (err == -EEXIST) {
1335 if (failed_start > start)
1336 clear_extent_bit(tree, start, failed_start - 1,
1337 EXTENT_LOCKED, 1, 0, NULL);
1343 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1345 unsigned long index = start >> PAGE_SHIFT;
1346 unsigned long end_index = end >> PAGE_SHIFT;
1349 while (index <= end_index) {
1350 page = find_get_page(inode->i_mapping, index);
1351 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1352 clear_page_dirty_for_io(page);
1358 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1360 unsigned long index = start >> PAGE_SHIFT;
1361 unsigned long end_index = end >> PAGE_SHIFT;
1364 while (index <= end_index) {
1365 page = find_get_page(inode->i_mapping, index);
1366 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1367 __set_page_dirty_nobuffers(page);
1368 account_page_redirty(page);
1374 /* find the first state struct with 'bits' set after 'start', and
1375 * return it. tree->lock must be held. NULL will returned if
1376 * nothing was found after 'start'
1378 static struct extent_state *
1379 find_first_extent_bit_state(struct extent_io_tree *tree,
1380 u64 start, unsigned bits)
1382 struct rb_node *node;
1383 struct extent_state *state;
1386 * this search will find all the extents that end after
1389 node = tree_search(tree, start);
1394 state = rb_entry(node, struct extent_state, rb_node);
1395 if (state->end >= start && (state->state & bits))
1398 node = rb_next(node);
1407 * find the first offset in the io tree with 'bits' set. zero is
1408 * returned if we find something, and *start_ret and *end_ret are
1409 * set to reflect the state struct that was found.
1411 * If nothing was found, 1 is returned. If found something, return 0.
1413 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1414 u64 *start_ret, u64 *end_ret, unsigned bits,
1415 struct extent_state **cached_state)
1417 struct extent_state *state;
1420 spin_lock(&tree->lock);
1421 if (cached_state && *cached_state) {
1422 state = *cached_state;
1423 if (state->end == start - 1 && extent_state_in_tree(state)) {
1424 while ((state = next_state(state)) != NULL) {
1425 if (state->state & bits)
1428 free_extent_state(*cached_state);
1429 *cached_state = NULL;
1432 free_extent_state(*cached_state);
1433 *cached_state = NULL;
1436 state = find_first_extent_bit_state(tree, start, bits);
1439 cache_state_if_flags(state, cached_state, 0);
1440 *start_ret = state->start;
1441 *end_ret = state->end;
1445 spin_unlock(&tree->lock);
1450 * find a contiguous range of bytes in the file marked as delalloc, not
1451 * more than 'max_bytes'. start and end are used to return the range,
1453 * true is returned if we find something, false if nothing was in the tree
1455 static noinline bool find_delalloc_range(struct extent_io_tree *tree,
1456 u64 *start, u64 *end, u64 max_bytes,
1457 struct extent_state **cached_state)
1459 struct rb_node *node;
1460 struct extent_state *state;
1461 u64 cur_start = *start;
1463 u64 total_bytes = 0;
1465 spin_lock(&tree->lock);
1468 * this search will find all the extents that end after
1471 node = tree_search(tree, cur_start);
1478 state = rb_entry(node, struct extent_state, rb_node);
1479 if (found && (state->start != cur_start ||
1480 (state->state & EXTENT_BOUNDARY))) {
1483 if (!(state->state & EXTENT_DELALLOC)) {
1489 *start = state->start;
1490 *cached_state = state;
1491 refcount_inc(&state->refs);
1495 cur_start = state->end + 1;
1496 node = rb_next(node);
1497 total_bytes += state->end - state->start + 1;
1498 if (total_bytes >= max_bytes)
1504 spin_unlock(&tree->lock);
1508 static int __process_pages_contig(struct address_space *mapping,
1509 struct page *locked_page,
1510 pgoff_t start_index, pgoff_t end_index,
1511 unsigned long page_ops, pgoff_t *index_ret);
1513 static noinline void __unlock_for_delalloc(struct inode *inode,
1514 struct page *locked_page,
1517 unsigned long index = start >> PAGE_SHIFT;
1518 unsigned long end_index = end >> PAGE_SHIFT;
1520 ASSERT(locked_page);
1521 if (index == locked_page->index && end_index == index)
1524 __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1528 static noinline int lock_delalloc_pages(struct inode *inode,
1529 struct page *locked_page,
1533 unsigned long index = delalloc_start >> PAGE_SHIFT;
1534 unsigned long index_ret = index;
1535 unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1538 ASSERT(locked_page);
1539 if (index == locked_page->index && index == end_index)
1542 ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1543 end_index, PAGE_LOCK, &index_ret);
1545 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1546 (u64)index_ret << PAGE_SHIFT);
1551 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1552 * more than @max_bytes. @Start and @end are used to return the range,
1554 * Return: true if we find something
1555 * false if nothing was in the tree
1558 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
1559 struct extent_io_tree *tree,
1560 struct page *locked_page, u64 *start,
1563 u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
1567 struct extent_state *cached_state = NULL;
1572 /* step one, find a bunch of delalloc bytes starting at start */
1573 delalloc_start = *start;
1575 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1576 max_bytes, &cached_state);
1577 if (!found || delalloc_end <= *start) {
1578 *start = delalloc_start;
1579 *end = delalloc_end;
1580 free_extent_state(cached_state);
1585 * start comes from the offset of locked_page. We have to lock
1586 * pages in order, so we can't process delalloc bytes before
1589 if (delalloc_start < *start)
1590 delalloc_start = *start;
1593 * make sure to limit the number of pages we try to lock down
1595 if (delalloc_end + 1 - delalloc_start > max_bytes)
1596 delalloc_end = delalloc_start + max_bytes - 1;
1598 /* step two, lock all the pages after the page that has start */
1599 ret = lock_delalloc_pages(inode, locked_page,
1600 delalloc_start, delalloc_end);
1601 ASSERT(!ret || ret == -EAGAIN);
1602 if (ret == -EAGAIN) {
1603 /* some of the pages are gone, lets avoid looping by
1604 * shortening the size of the delalloc range we're searching
1606 free_extent_state(cached_state);
1607 cached_state = NULL;
1609 max_bytes = PAGE_SIZE;
1618 /* step three, lock the state bits for the whole range */
1619 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1621 /* then test to make sure it is all still delalloc */
1622 ret = test_range_bit(tree, delalloc_start, delalloc_end,
1623 EXTENT_DELALLOC, 1, cached_state);
1625 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1627 __unlock_for_delalloc(inode, locked_page,
1628 delalloc_start, delalloc_end);
1632 free_extent_state(cached_state);
1633 *start = delalloc_start;
1634 *end = delalloc_end;
1639 static int __process_pages_contig(struct address_space *mapping,
1640 struct page *locked_page,
1641 pgoff_t start_index, pgoff_t end_index,
1642 unsigned long page_ops, pgoff_t *index_ret)
1644 unsigned long nr_pages = end_index - start_index + 1;
1645 unsigned long pages_locked = 0;
1646 pgoff_t index = start_index;
1647 struct page *pages[16];
1652 if (page_ops & PAGE_LOCK) {
1653 ASSERT(page_ops == PAGE_LOCK);
1654 ASSERT(index_ret && *index_ret == start_index);
1657 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1658 mapping_set_error(mapping, -EIO);
1660 while (nr_pages > 0) {
1661 ret = find_get_pages_contig(mapping, index,
1662 min_t(unsigned long,
1663 nr_pages, ARRAY_SIZE(pages)), pages);
1666 * Only if we're going to lock these pages,
1667 * can we find nothing at @index.
1669 ASSERT(page_ops & PAGE_LOCK);
1674 for (i = 0; i < ret; i++) {
1675 if (page_ops & PAGE_SET_PRIVATE2)
1676 SetPagePrivate2(pages[i]);
1678 if (pages[i] == locked_page) {
1683 if (page_ops & PAGE_CLEAR_DIRTY)
1684 clear_page_dirty_for_io(pages[i]);
1685 if (page_ops & PAGE_SET_WRITEBACK)
1686 set_page_writeback(pages[i]);
1687 if (page_ops & PAGE_SET_ERROR)
1688 SetPageError(pages[i]);
1689 if (page_ops & PAGE_END_WRITEBACK)
1690 end_page_writeback(pages[i]);
1691 if (page_ops & PAGE_UNLOCK)
1692 unlock_page(pages[i]);
1693 if (page_ops & PAGE_LOCK) {
1694 lock_page(pages[i]);
1695 if (!PageDirty(pages[i]) ||
1696 pages[i]->mapping != mapping) {
1697 unlock_page(pages[i]);
1711 if (err && index_ret)
1712 *index_ret = start_index + pages_locked - 1;
1716 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1717 u64 delalloc_end, struct page *locked_page,
1718 unsigned clear_bits,
1719 unsigned long page_ops)
1721 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
1724 __process_pages_contig(inode->i_mapping, locked_page,
1725 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
1730 * count the number of bytes in the tree that have a given bit(s)
1731 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1732 * cached. The total number found is returned.
1734 u64 count_range_bits(struct extent_io_tree *tree,
1735 u64 *start, u64 search_end, u64 max_bytes,
1736 unsigned bits, int contig)
1738 struct rb_node *node;
1739 struct extent_state *state;
1740 u64 cur_start = *start;
1741 u64 total_bytes = 0;
1745 if (WARN_ON(search_end <= cur_start))
1748 spin_lock(&tree->lock);
1749 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1750 total_bytes = tree->dirty_bytes;
1754 * this search will find all the extents that end after
1757 node = tree_search(tree, cur_start);
1762 state = rb_entry(node, struct extent_state, rb_node);
1763 if (state->start > search_end)
1765 if (contig && found && state->start > last + 1)
1767 if (state->end >= cur_start && (state->state & bits) == bits) {
1768 total_bytes += min(search_end, state->end) + 1 -
1769 max(cur_start, state->start);
1770 if (total_bytes >= max_bytes)
1773 *start = max(cur_start, state->start);
1777 } else if (contig && found) {
1780 node = rb_next(node);
1785 spin_unlock(&tree->lock);
1790 * set the private field for a given byte offset in the tree. If there isn't
1791 * an extent_state there already, this does nothing.
1793 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
1794 struct io_failure_record *failrec)
1796 struct rb_node *node;
1797 struct extent_state *state;
1800 spin_lock(&tree->lock);
1802 * this search will find all the extents that end after
1805 node = tree_search(tree, start);
1810 state = rb_entry(node, struct extent_state, rb_node);
1811 if (state->start != start) {
1815 state->failrec = failrec;
1817 spin_unlock(&tree->lock);
1821 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
1822 struct io_failure_record **failrec)
1824 struct rb_node *node;
1825 struct extent_state *state;
1828 spin_lock(&tree->lock);
1830 * this search will find all the extents that end after
1833 node = tree_search(tree, start);
1838 state = rb_entry(node, struct extent_state, rb_node);
1839 if (state->start != start) {
1843 *failrec = state->failrec;
1845 spin_unlock(&tree->lock);
1850 * searches a range in the state tree for a given mask.
1851 * If 'filled' == 1, this returns 1 only if every extent in the tree
1852 * has the bits set. Otherwise, 1 is returned if any bit in the
1853 * range is found set.
1855 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1856 unsigned bits, int filled, struct extent_state *cached)
1858 struct extent_state *state = NULL;
1859 struct rb_node *node;
1862 spin_lock(&tree->lock);
1863 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
1864 cached->end > start)
1865 node = &cached->rb_node;
1867 node = tree_search(tree, start);
1868 while (node && start <= end) {
1869 state = rb_entry(node, struct extent_state, rb_node);
1871 if (filled && state->start > start) {
1876 if (state->start > end)
1879 if (state->state & bits) {
1883 } else if (filled) {
1888 if (state->end == (u64)-1)
1891 start = state->end + 1;
1894 node = rb_next(node);
1901 spin_unlock(&tree->lock);
1906 * helper function to set a given page up to date if all the
1907 * extents in the tree for that page are up to date
1909 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1911 u64 start = page_offset(page);
1912 u64 end = start + PAGE_SIZE - 1;
1913 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1914 SetPageUptodate(page);
1917 int free_io_failure(struct extent_io_tree *failure_tree,
1918 struct extent_io_tree *io_tree,
1919 struct io_failure_record *rec)
1924 set_state_failrec(failure_tree, rec->start, NULL);
1925 ret = clear_extent_bits(failure_tree, rec->start,
1926 rec->start + rec->len - 1,
1927 EXTENT_LOCKED | EXTENT_DIRTY);
1931 ret = clear_extent_bits(io_tree, rec->start,
1932 rec->start + rec->len - 1,
1942 * this bypasses the standard btrfs submit functions deliberately, as
1943 * the standard behavior is to write all copies in a raid setup. here we only
1944 * want to write the one bad copy. so we do the mapping for ourselves and issue
1945 * submit_bio directly.
1946 * to avoid any synchronization issues, wait for the data after writing, which
1947 * actually prevents the read that triggered the error from finishing.
1948 * currently, there can be no more than two copies of every data bit. thus,
1949 * exactly one rewrite is required.
1951 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1952 u64 length, u64 logical, struct page *page,
1953 unsigned int pg_offset, int mirror_num)
1956 struct btrfs_device *dev;
1959 struct btrfs_bio *bbio = NULL;
1962 ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
1963 BUG_ON(!mirror_num);
1965 bio = btrfs_io_bio_alloc(1);
1966 bio->bi_iter.bi_size = 0;
1967 map_length = length;
1970 * Avoid races with device replace and make sure our bbio has devices
1971 * associated to its stripes that don't go away while we are doing the
1972 * read repair operation.
1974 btrfs_bio_counter_inc_blocked(fs_info);
1975 if (btrfs_is_parity_mirror(fs_info, logical, length)) {
1977 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
1978 * to update all raid stripes, but here we just want to correct
1979 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
1980 * stripe's dev and sector.
1982 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
1983 &map_length, &bbio, 0);
1985 btrfs_bio_counter_dec(fs_info);
1989 ASSERT(bbio->mirror_num == 1);
1991 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
1992 &map_length, &bbio, mirror_num);
1994 btrfs_bio_counter_dec(fs_info);
1998 BUG_ON(mirror_num != bbio->mirror_num);
2001 sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2002 bio->bi_iter.bi_sector = sector;
2003 dev = bbio->stripes[bbio->mirror_num - 1].dev;
2004 btrfs_put_bbio(bbio);
2005 if (!dev || !dev->bdev ||
2006 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2007 btrfs_bio_counter_dec(fs_info);
2011 bio_set_dev(bio, dev->bdev);
2012 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2013 bio_add_page(bio, page, length, pg_offset);
2015 if (btrfsic_submit_bio_wait(bio)) {
2016 /* try to remap that extent elsewhere? */
2017 btrfs_bio_counter_dec(fs_info);
2019 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2023 btrfs_info_rl_in_rcu(fs_info,
2024 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2026 rcu_str_deref(dev->name), sector);
2027 btrfs_bio_counter_dec(fs_info);
2032 int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2033 struct extent_buffer *eb, int mirror_num)
2035 u64 start = eb->start;
2036 int i, num_pages = num_extent_pages(eb);
2039 if (sb_rdonly(fs_info->sb))
2042 for (i = 0; i < num_pages; i++) {
2043 struct page *p = eb->pages[i];
2045 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2046 start - page_offset(p), mirror_num);
2056 * each time an IO finishes, we do a fast check in the IO failure tree
2057 * to see if we need to process or clean up an io_failure_record
2059 int clean_io_failure(struct btrfs_fs_info *fs_info,
2060 struct extent_io_tree *failure_tree,
2061 struct extent_io_tree *io_tree, u64 start,
2062 struct page *page, u64 ino, unsigned int pg_offset)
2065 struct io_failure_record *failrec;
2066 struct extent_state *state;
2071 ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2076 ret = get_state_failrec(failure_tree, start, &failrec);
2080 BUG_ON(!failrec->this_mirror);
2082 if (failrec->in_validation) {
2083 /* there was no real error, just free the record */
2084 btrfs_debug(fs_info,
2085 "clean_io_failure: freeing dummy error at %llu",
2089 if (sb_rdonly(fs_info->sb))
2092 spin_lock(&io_tree->lock);
2093 state = find_first_extent_bit_state(io_tree,
2096 spin_unlock(&io_tree->lock);
2098 if (state && state->start <= failrec->start &&
2099 state->end >= failrec->start + failrec->len - 1) {
2100 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2102 if (num_copies > 1) {
2103 repair_io_failure(fs_info, ino, start, failrec->len,
2104 failrec->logical, page, pg_offset,
2105 failrec->failed_mirror);
2110 free_io_failure(failure_tree, io_tree, failrec);
2116 * Can be called when
2117 * - hold extent lock
2118 * - under ordered extent
2119 * - the inode is freeing
2121 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2123 struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2124 struct io_failure_record *failrec;
2125 struct extent_state *state, *next;
2127 if (RB_EMPTY_ROOT(&failure_tree->state))
2130 spin_lock(&failure_tree->lock);
2131 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2133 if (state->start > end)
2136 ASSERT(state->end <= end);
2138 next = next_state(state);
2140 failrec = state->failrec;
2141 free_extent_state(state);
2146 spin_unlock(&failure_tree->lock);
2149 int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2150 struct io_failure_record **failrec_ret)
2152 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2153 struct io_failure_record *failrec;
2154 struct extent_map *em;
2155 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2156 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2157 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2161 ret = get_state_failrec(failure_tree, start, &failrec);
2163 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2167 failrec->start = start;
2168 failrec->len = end - start + 1;
2169 failrec->this_mirror = 0;
2170 failrec->bio_flags = 0;
2171 failrec->in_validation = 0;
2173 read_lock(&em_tree->lock);
2174 em = lookup_extent_mapping(em_tree, start, failrec->len);
2176 read_unlock(&em_tree->lock);
2181 if (em->start > start || em->start + em->len <= start) {
2182 free_extent_map(em);
2185 read_unlock(&em_tree->lock);
2191 logical = start - em->start;
2192 logical = em->block_start + logical;
2193 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2194 logical = em->block_start;
2195 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2196 extent_set_compress_type(&failrec->bio_flags,
2200 btrfs_debug(fs_info,
2201 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2202 logical, start, failrec->len);
2204 failrec->logical = logical;
2205 free_extent_map(em);
2207 /* set the bits in the private failure tree */
2208 ret = set_extent_bits(failure_tree, start, end,
2209 EXTENT_LOCKED | EXTENT_DIRTY);
2211 ret = set_state_failrec(failure_tree, start, failrec);
2212 /* set the bits in the inode's tree */
2214 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2220 btrfs_debug(fs_info,
2221 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2222 failrec->logical, failrec->start, failrec->len,
2223 failrec->in_validation);
2225 * when data can be on disk more than twice, add to failrec here
2226 * (e.g. with a list for failed_mirror) to make
2227 * clean_io_failure() clean all those errors at once.
2231 *failrec_ret = failrec;
2236 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2237 struct io_failure_record *failrec, int failed_mirror)
2239 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2242 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2243 if (num_copies == 1) {
2245 * we only have a single copy of the data, so don't bother with
2246 * all the retry and error correction code that follows. no
2247 * matter what the error is, it is very likely to persist.
2249 btrfs_debug(fs_info,
2250 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2251 num_copies, failrec->this_mirror, failed_mirror);
2256 * there are two premises:
2257 * a) deliver good data to the caller
2258 * b) correct the bad sectors on disk
2260 if (failed_bio_pages > 1) {
2262 * to fulfill b), we need to know the exact failing sectors, as
2263 * we don't want to rewrite any more than the failed ones. thus,
2264 * we need separate read requests for the failed bio
2266 * if the following BUG_ON triggers, our validation request got
2267 * merged. we need separate requests for our algorithm to work.
2269 BUG_ON(failrec->in_validation);
2270 failrec->in_validation = 1;
2271 failrec->this_mirror = failed_mirror;
2274 * we're ready to fulfill a) and b) alongside. get a good copy
2275 * of the failed sector and if we succeed, we have setup
2276 * everything for repair_io_failure to do the rest for us.
2278 if (failrec->in_validation) {
2279 BUG_ON(failrec->this_mirror != failed_mirror);
2280 failrec->in_validation = 0;
2281 failrec->this_mirror = 0;
2283 failrec->failed_mirror = failed_mirror;
2284 failrec->this_mirror++;
2285 if (failrec->this_mirror == failed_mirror)
2286 failrec->this_mirror++;
2289 if (failrec->this_mirror > num_copies) {
2290 btrfs_debug(fs_info,
2291 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2292 num_copies, failrec->this_mirror, failed_mirror);
2300 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2301 struct io_failure_record *failrec,
2302 struct page *page, int pg_offset, int icsum,
2303 bio_end_io_t *endio_func, void *data)
2305 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2307 struct btrfs_io_bio *btrfs_failed_bio;
2308 struct btrfs_io_bio *btrfs_bio;
2310 bio = btrfs_io_bio_alloc(1);
2311 bio->bi_end_io = endio_func;
2312 bio->bi_iter.bi_sector = failrec->logical >> 9;
2313 bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
2314 bio->bi_iter.bi_size = 0;
2315 bio->bi_private = data;
2317 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2318 if (btrfs_failed_bio->csum) {
2319 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2321 btrfs_bio = btrfs_io_bio(bio);
2322 btrfs_bio->csum = btrfs_bio->csum_inline;
2324 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2328 bio_add_page(bio, page, failrec->len, pg_offset);
2334 * This is a generic handler for readpage errors. If other copies exist, read
2335 * those and write back good data to the failed position. Does not investigate
2336 * in remapping the failed extent elsewhere, hoping the device will be smart
2337 * enough to do this as needed
2339 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2340 struct page *page, u64 start, u64 end,
2343 struct io_failure_record *failrec;
2344 struct inode *inode = page->mapping->host;
2345 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2346 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2349 blk_status_t status;
2351 unsigned failed_bio_pages = bio_pages_all(failed_bio);
2353 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2355 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2359 if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2361 free_io_failure(failure_tree, tree, failrec);
2365 if (failed_bio_pages > 1)
2366 read_mode |= REQ_FAILFAST_DEV;
2368 phy_offset >>= inode->i_sb->s_blocksize_bits;
2369 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2370 start - page_offset(page),
2371 (int)phy_offset, failed_bio->bi_end_io,
2373 bio->bi_opf = REQ_OP_READ | read_mode;
2375 btrfs_debug(btrfs_sb(inode->i_sb),
2376 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2377 read_mode, failrec->this_mirror, failrec->in_validation);
2379 status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2380 failrec->bio_flags, 0);
2382 free_io_failure(failure_tree, tree, failrec);
2384 ret = blk_status_to_errno(status);
2390 /* lots and lots of room for performance fixes in the end_bio funcs */
2392 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2394 int uptodate = (err == 0);
2397 btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
2400 ClearPageUptodate(page);
2402 ret = err < 0 ? err : -EIO;
2403 mapping_set_error(page->mapping, ret);
2408 * after a writepage IO is done, we need to:
2409 * clear the uptodate bits on error
2410 * clear the writeback bits in the extent tree for this IO
2411 * end_page_writeback if the page has no more pending IO
2413 * Scheduling is not allowed, so the extent state tree is expected
2414 * to have one and only one object corresponding to this IO.
2416 static void end_bio_extent_writepage(struct bio *bio)
2418 int error = blk_status_to_errno(bio->bi_status);
2419 struct bio_vec *bvec;
2424 ASSERT(!bio_flagged(bio, BIO_CLONED));
2425 bio_for_each_segment_all(bvec, bio, i) {
2426 struct page *page = bvec->bv_page;
2427 struct inode *inode = page->mapping->host;
2428 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2430 /* We always issue full-page reads, but if some block
2431 * in a page fails to read, blk_update_request() will
2432 * advance bv_offset and adjust bv_len to compensate.
2433 * Print a warning for nonzero offsets, and an error
2434 * if they don't add up to a full page. */
2435 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2436 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2438 "partial page write in btrfs with offset %u and length %u",
2439 bvec->bv_offset, bvec->bv_len);
2442 "incomplete page write in btrfs with offset %u and length %u",
2443 bvec->bv_offset, bvec->bv_len);
2446 start = page_offset(page);
2447 end = start + bvec->bv_offset + bvec->bv_len - 1;
2449 end_extent_writepage(page, error, start, end);
2450 end_page_writeback(page);
2457 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2460 struct extent_state *cached = NULL;
2461 u64 end = start + len - 1;
2463 if (uptodate && tree->track_uptodate)
2464 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2465 unlock_extent_cached_atomic(tree, start, end, &cached);
2469 * after a readpage IO is done, we need to:
2470 * clear the uptodate bits on error
2471 * set the uptodate bits if things worked
2472 * set the page up to date if all extents in the tree are uptodate
2473 * clear the lock bit in the extent tree
2474 * unlock the page if there are no other extents locked for it
2476 * Scheduling is not allowed, so the extent state tree is expected
2477 * to have one and only one object corresponding to this IO.
2479 static void end_bio_extent_readpage(struct bio *bio)
2481 struct bio_vec *bvec;
2482 int uptodate = !bio->bi_status;
2483 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2484 struct extent_io_tree *tree, *failure_tree;
2489 u64 extent_start = 0;
2495 ASSERT(!bio_flagged(bio, BIO_CLONED));
2496 bio_for_each_segment_all(bvec, bio, i) {
2497 struct page *page = bvec->bv_page;
2498 struct inode *inode = page->mapping->host;
2499 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2500 bool data_inode = btrfs_ino(BTRFS_I(inode))
2501 != BTRFS_BTREE_INODE_OBJECTID;
2503 btrfs_debug(fs_info,
2504 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2505 (u64)bio->bi_iter.bi_sector, bio->bi_status,
2506 io_bio->mirror_num);
2507 tree = &BTRFS_I(inode)->io_tree;
2508 failure_tree = &BTRFS_I(inode)->io_failure_tree;
2510 /* We always issue full-page reads, but if some block
2511 * in a page fails to read, blk_update_request() will
2512 * advance bv_offset and adjust bv_len to compensate.
2513 * Print a warning for nonzero offsets, and an error
2514 * if they don't add up to a full page. */
2515 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2516 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2518 "partial page read in btrfs with offset %u and length %u",
2519 bvec->bv_offset, bvec->bv_len);
2522 "incomplete page read in btrfs with offset %u and length %u",
2523 bvec->bv_offset, bvec->bv_len);
2526 start = page_offset(page);
2527 end = start + bvec->bv_offset + bvec->bv_len - 1;
2530 mirror = io_bio->mirror_num;
2531 if (likely(uptodate)) {
2532 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2538 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2539 failure_tree, tree, start,
2541 btrfs_ino(BTRFS_I(inode)), 0);
2544 if (likely(uptodate))
2550 * The generic bio_readpage_error handles errors the
2551 * following way: If possible, new read requests are
2552 * created and submitted and will end up in
2553 * end_bio_extent_readpage as well (if we're lucky,
2554 * not in the !uptodate case). In that case it returns
2555 * 0 and we just go on with the next page in our bio.
2556 * If it can't handle the error it will return -EIO and
2557 * we remain responsible for that page.
2559 ret = bio_readpage_error(bio, offset, page, start, end,
2562 uptodate = !bio->bi_status;
2567 struct extent_buffer *eb;
2569 eb = (struct extent_buffer *)page->private;
2570 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2571 eb->read_mirror = mirror;
2572 atomic_dec(&eb->io_pages);
2573 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2575 btree_readahead_hook(eb, -EIO);
2580 if (likely(uptodate)) {
2581 loff_t i_size = i_size_read(inode);
2582 pgoff_t end_index = i_size >> PAGE_SHIFT;
2585 /* Zero out the end if this page straddles i_size */
2586 off = offset_in_page(i_size);
2587 if (page->index == end_index && off)
2588 zero_user_segment(page, off, PAGE_SIZE);
2589 SetPageUptodate(page);
2591 ClearPageUptodate(page);
2597 if (unlikely(!uptodate)) {
2599 endio_readpage_release_extent(tree,
2605 endio_readpage_release_extent(tree, start,
2606 end - start + 1, 0);
2607 } else if (!extent_len) {
2608 extent_start = start;
2609 extent_len = end + 1 - start;
2610 } else if (extent_start + extent_len == start) {
2611 extent_len += end + 1 - start;
2613 endio_readpage_release_extent(tree, extent_start,
2614 extent_len, uptodate);
2615 extent_start = start;
2616 extent_len = end + 1 - start;
2621 endio_readpage_release_extent(tree, extent_start, extent_len,
2623 btrfs_io_bio_free_csum(io_bio);
2628 * Initialize the members up to but not including 'bio'. Use after allocating a
2629 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2630 * 'bio' because use of __GFP_ZERO is not supported.
2632 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
2634 memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2638 * The following helpers allocate a bio. As it's backed by a bioset, it'll
2639 * never fail. We're returning a bio right now but you can call btrfs_io_bio
2640 * for the appropriate container_of magic
2642 struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
2646 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
2647 bio_set_dev(bio, bdev);
2648 bio->bi_iter.bi_sector = first_byte >> 9;
2649 btrfs_io_bio_init(btrfs_io_bio(bio));
2653 struct bio *btrfs_bio_clone(struct bio *bio)
2655 struct btrfs_io_bio *btrfs_bio;
2658 /* Bio allocation backed by a bioset does not fail */
2659 new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
2660 btrfs_bio = btrfs_io_bio(new);
2661 btrfs_io_bio_init(btrfs_bio);
2662 btrfs_bio->iter = bio->bi_iter;
2666 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
2670 /* Bio allocation backed by a bioset does not fail */
2671 bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
2672 btrfs_io_bio_init(btrfs_io_bio(bio));
2676 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
2679 struct btrfs_io_bio *btrfs_bio;
2681 /* this will never fail when it's backed by a bioset */
2682 bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
2685 btrfs_bio = btrfs_io_bio(bio);
2686 btrfs_io_bio_init(btrfs_bio);
2688 bio_trim(bio, offset >> 9, size >> 9);
2689 btrfs_bio->iter = bio->bi_iter;
2693 static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
2694 unsigned long bio_flags)
2696 blk_status_t ret = 0;
2697 struct bio_vec *bvec = bio_last_bvec_all(bio);
2698 struct page *page = bvec->bv_page;
2699 struct extent_io_tree *tree = bio->bi_private;
2702 start = page_offset(page) + bvec->bv_offset;
2704 bio->bi_private = NULL;
2707 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
2708 mirror_num, bio_flags, start);
2710 btrfsic_submit_bio(bio);
2712 return blk_status_to_errno(ret);
2716 * @opf: bio REQ_OP_* and REQ_* flags as one value
2717 * @tree: tree so we can call our merge_bio hook
2718 * @wbc: optional writeback control for io accounting
2719 * @page: page to add to the bio
2720 * @pg_offset: offset of the new bio or to check whether we are adding
2721 * a contiguous page to the previous one
2722 * @size: portion of page that we want to write
2723 * @offset: starting offset in the page
2724 * @bdev: attach newly created bios to this bdev
2725 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
2726 * @end_io_func: end_io callback for new bio
2727 * @mirror_num: desired mirror to read/write
2728 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2729 * @bio_flags: flags of the current bio to see if we can merge them
2731 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2732 struct writeback_control *wbc,
2733 struct page *page, u64 offset,
2734 size_t size, unsigned long pg_offset,
2735 struct block_device *bdev,
2736 struct bio **bio_ret,
2737 bio_end_io_t end_io_func,
2739 unsigned long prev_bio_flags,
2740 unsigned long bio_flags,
2741 bool force_bio_submit)
2745 size_t page_size = min_t(size_t, size, PAGE_SIZE);
2746 sector_t sector = offset >> 9;
2752 bool can_merge = true;
2755 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2756 contig = bio->bi_iter.bi_sector == sector;
2758 contig = bio_end_sector(bio) == sector;
2761 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
2764 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2766 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2767 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2775 wbc_account_io(wbc, page, page_size);
2780 bio = btrfs_bio_alloc(bdev, offset);
2781 bio_add_page(bio, page, page_size, pg_offset);
2782 bio->bi_end_io = end_io_func;
2783 bio->bi_private = tree;
2784 bio->bi_write_hint = page->mapping->host->i_write_hint;
2787 wbc_init_bio(wbc, bio);
2788 wbc_account_io(wbc, page, page_size);
2796 static void attach_extent_buffer_page(struct extent_buffer *eb,
2799 if (!PagePrivate(page)) {
2800 SetPagePrivate(page);
2802 set_page_private(page, (unsigned long)eb);
2804 WARN_ON(page->private != (unsigned long)eb);
2808 void set_page_extent_mapped(struct page *page)
2810 if (!PagePrivate(page)) {
2811 SetPagePrivate(page);
2813 set_page_private(page, EXTENT_PAGE_PRIVATE);
2817 static struct extent_map *
2818 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2819 u64 start, u64 len, get_extent_t *get_extent,
2820 struct extent_map **em_cached)
2822 struct extent_map *em;
2824 if (em_cached && *em_cached) {
2826 if (extent_map_in_tree(em) && start >= em->start &&
2827 start < extent_map_end(em)) {
2828 refcount_inc(&em->refs);
2832 free_extent_map(em);
2836 em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
2837 if (em_cached && !IS_ERR_OR_NULL(em)) {
2839 refcount_inc(&em->refs);
2845 * basic readpage implementation. Locked extent state structs are inserted
2846 * into the tree that are removed when the IO is done (by the end_io
2848 * XXX JDM: This needs looking at to ensure proper page locking
2849 * return 0 on success, otherwise return error
2851 static int __do_readpage(struct extent_io_tree *tree,
2853 get_extent_t *get_extent,
2854 struct extent_map **em_cached,
2855 struct bio **bio, int mirror_num,
2856 unsigned long *bio_flags, unsigned int read_flags,
2859 struct inode *inode = page->mapping->host;
2860 u64 start = page_offset(page);
2861 const u64 end = start + PAGE_SIZE - 1;
2864 u64 last_byte = i_size_read(inode);
2867 struct extent_map *em;
2868 struct block_device *bdev;
2871 size_t pg_offset = 0;
2873 size_t disk_io_size;
2874 size_t blocksize = inode->i_sb->s_blocksize;
2875 unsigned long this_bio_flag = 0;
2877 set_page_extent_mapped(page);
2879 if (!PageUptodate(page)) {
2880 if (cleancache_get_page(page) == 0) {
2881 BUG_ON(blocksize != PAGE_SIZE);
2882 unlock_extent(tree, start, end);
2887 if (page->index == last_byte >> PAGE_SHIFT) {
2889 size_t zero_offset = offset_in_page(last_byte);
2892 iosize = PAGE_SIZE - zero_offset;
2893 userpage = kmap_atomic(page);
2894 memset(userpage + zero_offset, 0, iosize);
2895 flush_dcache_page(page);
2896 kunmap_atomic(userpage);
2899 while (cur <= end) {
2900 bool force_bio_submit = false;
2903 if (cur >= last_byte) {
2905 struct extent_state *cached = NULL;
2907 iosize = PAGE_SIZE - pg_offset;
2908 userpage = kmap_atomic(page);
2909 memset(userpage + pg_offset, 0, iosize);
2910 flush_dcache_page(page);
2911 kunmap_atomic(userpage);
2912 set_extent_uptodate(tree, cur, cur + iosize - 1,
2914 unlock_extent_cached(tree, cur,
2915 cur + iosize - 1, &cached);
2918 em = __get_extent_map(inode, page, pg_offset, cur,
2919 end - cur + 1, get_extent, em_cached);
2920 if (IS_ERR_OR_NULL(em)) {
2922 unlock_extent(tree, cur, end);
2925 extent_offset = cur - em->start;
2926 BUG_ON(extent_map_end(em) <= cur);
2929 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2930 this_bio_flag |= EXTENT_BIO_COMPRESSED;
2931 extent_set_compress_type(&this_bio_flag,
2935 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2936 cur_end = min(extent_map_end(em) - 1, end);
2937 iosize = ALIGN(iosize, blocksize);
2938 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2939 disk_io_size = em->block_len;
2940 offset = em->block_start;
2942 offset = em->block_start + extent_offset;
2943 disk_io_size = iosize;
2946 block_start = em->block_start;
2947 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2948 block_start = EXTENT_MAP_HOLE;
2951 * If we have a file range that points to a compressed extent
2952 * and it's followed by a consecutive file range that points to
2953 * to the same compressed extent (possibly with a different
2954 * offset and/or length, so it either points to the whole extent
2955 * or only part of it), we must make sure we do not submit a
2956 * single bio to populate the pages for the 2 ranges because
2957 * this makes the compressed extent read zero out the pages
2958 * belonging to the 2nd range. Imagine the following scenario:
2961 * [0 - 8K] [8K - 24K]
2964 * points to extent X, points to extent X,
2965 * offset 4K, length of 8K offset 0, length 16K
2967 * [extent X, compressed length = 4K uncompressed length = 16K]
2969 * If the bio to read the compressed extent covers both ranges,
2970 * it will decompress extent X into the pages belonging to the
2971 * first range and then it will stop, zeroing out the remaining
2972 * pages that belong to the other range that points to extent X.
2973 * So here we make sure we submit 2 bios, one for the first
2974 * range and another one for the third range. Both will target
2975 * the same physical extent from disk, but we can't currently
2976 * make the compressed bio endio callback populate the pages
2977 * for both ranges because each compressed bio is tightly
2978 * coupled with a single extent map, and each range can have
2979 * an extent map with a different offset value relative to the
2980 * uncompressed data of our extent and different lengths. This
2981 * is a corner case so we prioritize correctness over
2982 * non-optimal behavior (submitting 2 bios for the same extent).
2984 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
2985 prev_em_start && *prev_em_start != (u64)-1 &&
2986 *prev_em_start != em->orig_start)
2987 force_bio_submit = true;
2990 *prev_em_start = em->orig_start;
2992 free_extent_map(em);
2995 /* we've found a hole, just zero and go on */
2996 if (block_start == EXTENT_MAP_HOLE) {
2998 struct extent_state *cached = NULL;
3000 userpage = kmap_atomic(page);
3001 memset(userpage + pg_offset, 0, iosize);
3002 flush_dcache_page(page);
3003 kunmap_atomic(userpage);
3005 set_extent_uptodate(tree, cur, cur + iosize - 1,
3007 unlock_extent_cached(tree, cur,
3008 cur + iosize - 1, &cached);
3010 pg_offset += iosize;
3013 /* the get_extent function already copied into the page */
3014 if (test_range_bit(tree, cur, cur_end,
3015 EXTENT_UPTODATE, 1, NULL)) {
3016 check_page_uptodate(tree, page);
3017 unlock_extent(tree, cur, cur + iosize - 1);
3019 pg_offset += iosize;
3022 /* we have an inline extent but it didn't get marked up
3023 * to date. Error out
3025 if (block_start == EXTENT_MAP_INLINE) {
3027 unlock_extent(tree, cur, cur + iosize - 1);
3029 pg_offset += iosize;
3033 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
3034 page, offset, disk_io_size,
3035 pg_offset, bdev, bio,
3036 end_bio_extent_readpage, mirror_num,
3042 *bio_flags = this_bio_flag;
3045 unlock_extent(tree, cur, cur + iosize - 1);
3049 pg_offset += iosize;
3053 if (!PageError(page))
3054 SetPageUptodate(page);
3060 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3061 struct page *pages[], int nr_pages,
3063 struct extent_map **em_cached,
3065 unsigned long *bio_flags,
3068 struct inode *inode;
3069 struct btrfs_ordered_extent *ordered;
3072 inode = pages[0]->mapping->host;
3074 lock_extent(tree, start, end);
3075 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
3079 unlock_extent(tree, start, end);
3080 btrfs_start_ordered_extent(inode, ordered, 1);
3081 btrfs_put_ordered_extent(ordered);
3084 for (index = 0; index < nr_pages; index++) {
3085 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
3086 bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
3087 put_page(pages[index]);
3091 static void __extent_readpages(struct extent_io_tree *tree,
3092 struct page *pages[],
3094 struct extent_map **em_cached,
3095 struct bio **bio, unsigned long *bio_flags,
3102 int first_index = 0;
3104 for (index = 0; index < nr_pages; index++) {
3105 page_start = page_offset(pages[index]);
3108 end = start + PAGE_SIZE - 1;
3109 first_index = index;
3110 } else if (end + 1 == page_start) {
3113 __do_contiguous_readpages(tree, &pages[first_index],
3114 index - first_index, start,