btrfs: lift errors from add_extent_changeset to the callers
[sfrench/cifs-2.6.git] / fs / btrfs / extent_io.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/bitops.h>
3 #include <linux/slab.h>
4 #include <linux/bio.h>
5 #include <linux/mm.h>
6 #include <linux/pagemap.h>
7 #include <linux/page-flags.h>
8 #include <linux/spinlock.h>
9 #include <linux/blkdev.h>
10 #include <linux/swap.h>
11 #include <linux/writeback.h>
12 #include <linux/pagevec.h>
13 #include <linux/prefetch.h>
14 #include <linux/cleancache.h>
15 #include "extent_io.h"
16 #include "extent_map.h"
17 #include "ctree.h"
18 #include "btrfs_inode.h"
19 #include "volumes.h"
20 #include "check-integrity.h"
21 #include "locking.h"
22 #include "rcu-string.h"
23 #include "backref.h"
24 #include "disk-io.h"
25
26 static struct kmem_cache *extent_state_cache;
27 static struct kmem_cache *extent_buffer_cache;
28 static struct bio_set *btrfs_bioset;
29
30 static inline bool extent_state_in_tree(const struct extent_state *state)
31 {
32         return !RB_EMPTY_NODE(&state->rb_node);
33 }
34
35 #ifdef CONFIG_BTRFS_DEBUG
36 static LIST_HEAD(buffers);
37 static LIST_HEAD(states);
38
39 static DEFINE_SPINLOCK(leak_lock);
40
41 static inline
42 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
43 {
44         unsigned long flags;
45
46         spin_lock_irqsave(&leak_lock, flags);
47         list_add(new, head);
48         spin_unlock_irqrestore(&leak_lock, flags);
49 }
50
51 static inline
52 void btrfs_leak_debug_del(struct list_head *entry)
53 {
54         unsigned long flags;
55
56         spin_lock_irqsave(&leak_lock, flags);
57         list_del(entry);
58         spin_unlock_irqrestore(&leak_lock, flags);
59 }
60
61 static inline
62 void btrfs_leak_debug_check(void)
63 {
64         struct extent_state *state;
65         struct extent_buffer *eb;
66
67         while (!list_empty(&states)) {
68                 state = list_entry(states.next, struct extent_state, leak_list);
69                 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
70                        state->start, state->end, state->state,
71                        extent_state_in_tree(state),
72                        refcount_read(&state->refs));
73                 list_del(&state->leak_list);
74                 kmem_cache_free(extent_state_cache, state);
75         }
76
77         while (!list_empty(&buffers)) {
78                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
79                 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
80                        eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
81                 list_del(&eb->leak_list);
82                 kmem_cache_free(extent_buffer_cache, eb);
83         }
84 }
85
86 #define btrfs_debug_check_extent_io_range(tree, start, end)             \
87         __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
88 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
89                 struct extent_io_tree *tree, u64 start, u64 end)
90 {
91         if (tree->ops && tree->ops->check_extent_io_range)
92                 tree->ops->check_extent_io_range(tree->private_data, caller,
93                                                  start, end);
94 }
95 #else
96 #define btrfs_leak_debug_add(new, head) do {} while (0)
97 #define btrfs_leak_debug_del(entry)     do {} while (0)
98 #define btrfs_leak_debug_check()        do {} while (0)
99 #define btrfs_debug_check_extent_io_range(c, s, e)      do {} while (0)
100 #endif
101
102 #define BUFFER_LRU_MAX 64
103
104 struct tree_entry {
105         u64 start;
106         u64 end;
107         struct rb_node rb_node;
108 };
109
110 struct extent_page_data {
111         struct bio *bio;
112         struct extent_io_tree *tree;
113         /* tells writepage not to lock the state bits for this range
114          * it still does the unlocking
115          */
116         unsigned int extent_locked:1;
117
118         /* tells the submit_bio code to use REQ_SYNC */
119         unsigned int sync_io:1;
120 };
121
122 static int add_extent_changeset(struct extent_state *state, unsigned bits,
123                                  struct extent_changeset *changeset,
124                                  int set)
125 {
126         int ret;
127
128         if (!changeset)
129                 return 0;
130         if (set && (state->state & bits) == bits)
131                 return 0;
132         if (!set && (state->state & bits) == 0)
133                 return 0;
134         changeset->bytes_changed += state->end - state->start + 1;
135         ret = ulist_add(&changeset->range_changed, state->start, state->end,
136                         GFP_ATOMIC);
137         return ret;
138 }
139
140 static void flush_write_bio(struct extent_page_data *epd);
141
142 static inline struct btrfs_fs_info *
143 tree_fs_info(struct extent_io_tree *tree)
144 {
145         if (tree->ops)
146                 return tree->ops->tree_fs_info(tree->private_data);
147         return NULL;
148 }
149
150 int __init extent_io_init(void)
151 {
152         extent_state_cache = kmem_cache_create("btrfs_extent_state",
153                         sizeof(struct extent_state), 0,
154                         SLAB_MEM_SPREAD, NULL);
155         if (!extent_state_cache)
156                 return -ENOMEM;
157
158         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
159                         sizeof(struct extent_buffer), 0,
160                         SLAB_MEM_SPREAD, NULL);
161         if (!extent_buffer_cache)
162                 goto free_state_cache;
163
164         btrfs_bioset = bioset_create(BIO_POOL_SIZE,
165                                      offsetof(struct btrfs_io_bio, bio),
166                                      BIOSET_NEED_BVECS);
167         if (!btrfs_bioset)
168                 goto free_buffer_cache;
169
170         if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
171                 goto free_bioset;
172
173         return 0;
174
175 free_bioset:
176         bioset_free(btrfs_bioset);
177         btrfs_bioset = NULL;
178
179 free_buffer_cache:
180         kmem_cache_destroy(extent_buffer_cache);
181         extent_buffer_cache = NULL;
182
183 free_state_cache:
184         kmem_cache_destroy(extent_state_cache);
185         extent_state_cache = NULL;
186         return -ENOMEM;
187 }
188
189 void __cold extent_io_exit(void)
190 {
191         btrfs_leak_debug_check();
192
193         /*
194          * Make sure all delayed rcu free are flushed before we
195          * destroy caches.
196          */
197         rcu_barrier();
198         kmem_cache_destroy(extent_state_cache);
199         kmem_cache_destroy(extent_buffer_cache);
200         if (btrfs_bioset)
201                 bioset_free(btrfs_bioset);
202 }
203
204 void extent_io_tree_init(struct extent_io_tree *tree,
205                          void *private_data)
206 {
207         tree->state = RB_ROOT;
208         tree->ops = NULL;
209         tree->dirty_bytes = 0;
210         spin_lock_init(&tree->lock);
211         tree->private_data = private_data;
212 }
213
214 static struct extent_state *alloc_extent_state(gfp_t mask)
215 {
216         struct extent_state *state;
217
218         /*
219          * The given mask might be not appropriate for the slab allocator,
220          * drop the unsupported bits
221          */
222         mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
223         state = kmem_cache_alloc(extent_state_cache, mask);
224         if (!state)
225                 return state;
226         state->state = 0;
227         state->failrec = NULL;
228         RB_CLEAR_NODE(&state->rb_node);
229         btrfs_leak_debug_add(&state->leak_list, &states);
230         refcount_set(&state->refs, 1);
231         init_waitqueue_head(&state->wq);
232         trace_alloc_extent_state(state, mask, _RET_IP_);
233         return state;
234 }
235
236 void free_extent_state(struct extent_state *state)
237 {
238         if (!state)
239                 return;
240         if (refcount_dec_and_test(&state->refs)) {
241                 WARN_ON(extent_state_in_tree(state));
242                 btrfs_leak_debug_del(&state->leak_list);
243                 trace_free_extent_state(state, _RET_IP_);
244                 kmem_cache_free(extent_state_cache, state);
245         }
246 }
247
248 static struct rb_node *tree_insert(struct rb_root *root,
249                                    struct rb_node *search_start,
250                                    u64 offset,
251                                    struct rb_node *node,
252                                    struct rb_node ***p_in,
253                                    struct rb_node **parent_in)
254 {
255         struct rb_node **p;
256         struct rb_node *parent = NULL;
257         struct tree_entry *entry;
258
259         if (p_in && parent_in) {
260                 p = *p_in;
261                 parent = *parent_in;
262                 goto do_insert;
263         }
264
265         p = search_start ? &search_start : &root->rb_node;
266         while (*p) {
267                 parent = *p;
268                 entry = rb_entry(parent, struct tree_entry, rb_node);
269
270                 if (offset < entry->start)
271                         p = &(*p)->rb_left;
272                 else if (offset > entry->end)
273                         p = &(*p)->rb_right;
274                 else
275                         return parent;
276         }
277
278 do_insert:
279         rb_link_node(node, parent, p);
280         rb_insert_color(node, root);
281         return NULL;
282 }
283
284 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
285                                       struct rb_node **prev_ret,
286                                       struct rb_node **next_ret,
287                                       struct rb_node ***p_ret,
288                                       struct rb_node **parent_ret)
289 {
290         struct rb_root *root = &tree->state;
291         struct rb_node **n = &root->rb_node;
292         struct rb_node *prev = NULL;
293         struct rb_node *orig_prev = NULL;
294         struct tree_entry *entry;
295         struct tree_entry *prev_entry = NULL;
296
297         while (*n) {
298                 prev = *n;
299                 entry = rb_entry(prev, struct tree_entry, rb_node);
300                 prev_entry = entry;
301
302                 if (offset < entry->start)
303                         n = &(*n)->rb_left;
304                 else if (offset > entry->end)
305                         n = &(*n)->rb_right;
306                 else
307                         return *n;
308         }
309
310         if (p_ret)
311                 *p_ret = n;
312         if (parent_ret)
313                 *parent_ret = prev;
314
315         if (prev_ret) {
316                 orig_prev = prev;
317                 while (prev && offset > prev_entry->end) {
318                         prev = rb_next(prev);
319                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
320                 }
321                 *prev_ret = prev;
322                 prev = orig_prev;
323         }
324
325         if (next_ret) {
326                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
327                 while (prev && offset < prev_entry->start) {
328                         prev = rb_prev(prev);
329                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
330                 }
331                 *next_ret = prev;
332         }
333         return NULL;
334 }
335
336 static inline struct rb_node *
337 tree_search_for_insert(struct extent_io_tree *tree,
338                        u64 offset,
339                        struct rb_node ***p_ret,
340                        struct rb_node **parent_ret)
341 {
342         struct rb_node *prev = NULL;
343         struct rb_node *ret;
344
345         ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
346         if (!ret)
347                 return prev;
348         return ret;
349 }
350
351 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
352                                           u64 offset)
353 {
354         return tree_search_for_insert(tree, offset, NULL, NULL);
355 }
356
357 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
358                      struct extent_state *other)
359 {
360         if (tree->ops && tree->ops->merge_extent_hook)
361                 tree->ops->merge_extent_hook(tree->private_data, new, other);
362 }
363
364 /*
365  * utility function to look for merge candidates inside a given range.
366  * Any extents with matching state are merged together into a single
367  * extent in the tree.  Extents with EXTENT_IO in their state field
368  * are not merged because the end_io handlers need to be able to do
369  * operations on them without sleeping (or doing allocations/splits).
370  *
371  * This should be called with the tree lock held.
372  */
373 static void merge_state(struct extent_io_tree *tree,
374                         struct extent_state *state)
375 {
376         struct extent_state *other;
377         struct rb_node *other_node;
378
379         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
380                 return;
381
382         other_node = rb_prev(&state->rb_node);
383         if (other_node) {
384                 other = rb_entry(other_node, struct extent_state, rb_node);
385                 if (other->end == state->start - 1 &&
386                     other->state == state->state) {
387                         merge_cb(tree, state, other);
388                         state->start = other->start;
389                         rb_erase(&other->rb_node, &tree->state);
390                         RB_CLEAR_NODE(&other->rb_node);
391                         free_extent_state(other);
392                 }
393         }
394         other_node = rb_next(&state->rb_node);
395         if (other_node) {
396                 other = rb_entry(other_node, struct extent_state, rb_node);
397                 if (other->start == state->end + 1 &&
398                     other->state == state->state) {
399                         merge_cb(tree, state, other);
400                         state->end = other->end;
401                         rb_erase(&other->rb_node, &tree->state);
402                         RB_CLEAR_NODE(&other->rb_node);
403                         free_extent_state(other);
404                 }
405         }
406 }
407
408 static void set_state_cb(struct extent_io_tree *tree,
409                          struct extent_state *state, unsigned *bits)
410 {
411         if (tree->ops && tree->ops->set_bit_hook)
412                 tree->ops->set_bit_hook(tree->private_data, state, bits);
413 }
414
415 static void clear_state_cb(struct extent_io_tree *tree,
416                            struct extent_state *state, unsigned *bits)
417 {
418         if (tree->ops && tree->ops->clear_bit_hook)
419                 tree->ops->clear_bit_hook(tree->private_data, state, bits);
420 }
421
422 static void set_state_bits(struct extent_io_tree *tree,
423                            struct extent_state *state, unsigned *bits,
424                            struct extent_changeset *changeset);
425
426 /*
427  * insert an extent_state struct into the tree.  'bits' are set on the
428  * struct before it is inserted.
429  *
430  * This may return -EEXIST if the extent is already there, in which case the
431  * state struct is freed.
432  *
433  * The tree lock is not taken internally.  This is a utility function and
434  * probably isn't what you want to call (see set/clear_extent_bit).
435  */
436 static int insert_state(struct extent_io_tree *tree,
437                         struct extent_state *state, u64 start, u64 end,
438                         struct rb_node ***p,
439                         struct rb_node **parent,
440                         unsigned *bits, struct extent_changeset *changeset)
441 {
442         struct rb_node *node;
443
444         if (end < start)
445                 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
446                        end, start);
447         state->start = start;
448         state->end = end;
449
450         set_state_bits(tree, state, bits, changeset);
451
452         node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
453         if (node) {
454                 struct extent_state *found;
455                 found = rb_entry(node, struct extent_state, rb_node);
456                 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
457                        found->start, found->end, start, end);
458                 return -EEXIST;
459         }
460         merge_state(tree, state);
461         return 0;
462 }
463
464 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
465                      u64 split)
466 {
467         if (tree->ops && tree->ops->split_extent_hook)
468                 tree->ops->split_extent_hook(tree->private_data, orig, split);
469 }
470
471 /*
472  * split a given extent state struct in two, inserting the preallocated
473  * struct 'prealloc' as the newly created second half.  'split' indicates an
474  * offset inside 'orig' where it should be split.
475  *
476  * Before calling,
477  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
478  * are two extent state structs in the tree:
479  * prealloc: [orig->start, split - 1]
480  * orig: [ split, orig->end ]
481  *
482  * The tree locks are not taken by this function. They need to be held
483  * by the caller.
484  */
485 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
486                        struct extent_state *prealloc, u64 split)
487 {
488         struct rb_node *node;
489
490         split_cb(tree, orig, split);
491
492         prealloc->start = orig->start;
493         prealloc->end = split - 1;
494         prealloc->state = orig->state;
495         orig->start = split;
496
497         node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
498                            &prealloc->rb_node, NULL, NULL);
499         if (node) {
500                 free_extent_state(prealloc);
501                 return -EEXIST;
502         }
503         return 0;
504 }
505
506 static struct extent_state *next_state(struct extent_state *state)
507 {
508         struct rb_node *next = rb_next(&state->rb_node);
509         if (next)
510                 return rb_entry(next, struct extent_state, rb_node);
511         else
512                 return NULL;
513 }
514
515 /*
516  * utility function to clear some bits in an extent state struct.
517  * it will optionally wake up any one waiting on this state (wake == 1).
518  *
519  * If no bits are set on the state struct after clearing things, the
520  * struct is freed and removed from the tree
521  */
522 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
523                                             struct extent_state *state,
524                                             unsigned *bits, int wake,
525                                             struct extent_changeset *changeset)
526 {
527         struct extent_state *next;
528         unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
529         int ret;
530
531         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
532                 u64 range = state->end - state->start + 1;
533                 WARN_ON(range > tree->dirty_bytes);
534                 tree->dirty_bytes -= range;
535         }
536         clear_state_cb(tree, state, bits);
537         ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
538         BUG_ON(ret < 0);
539         state->state &= ~bits_to_clear;
540         if (wake)
541                 wake_up(&state->wq);
542         if (state->state == 0) {
543                 next = next_state(state);
544                 if (extent_state_in_tree(state)) {
545                         rb_erase(&state->rb_node, &tree->state);
546                         RB_CLEAR_NODE(&state->rb_node);
547                         free_extent_state(state);
548                 } else {
549                         WARN_ON(1);
550                 }
551         } else {
552                 merge_state(tree, state);
553                 next = next_state(state);
554         }
555         return next;
556 }
557
558 static struct extent_state *
559 alloc_extent_state_atomic(struct extent_state *prealloc)
560 {
561         if (!prealloc)
562                 prealloc = alloc_extent_state(GFP_ATOMIC);
563
564         return prealloc;
565 }
566
567 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
568 {
569         btrfs_panic(tree_fs_info(tree), err,
570                     "Locking error: Extent tree was modified by another thread while locked.");
571 }
572
573 /*
574  * clear some bits on a range in the tree.  This may require splitting
575  * or inserting elements in the tree, so the gfp mask is used to
576  * indicate which allocations or sleeping are allowed.
577  *
578  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
579  * the given range from the tree regardless of state (ie for truncate).
580  *
581  * the range [start, end] is inclusive.
582  *
583  * This takes the tree lock, and returns 0 on success and < 0 on error.
584  */
585 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
586                               unsigned bits, int wake, int delete,
587                               struct extent_state **cached_state,
588                               gfp_t mask, struct extent_changeset *changeset)
589 {
590         struct extent_state *state;
591         struct extent_state *cached;
592         struct extent_state *prealloc = NULL;
593         struct rb_node *node;
594         u64 last_end;
595         int err;
596         int clear = 0;
597
598         btrfs_debug_check_extent_io_range(tree, start, end);
599
600         if (bits & EXTENT_DELALLOC)
601                 bits |= EXTENT_NORESERVE;
602
603         if (delete)
604                 bits |= ~EXTENT_CTLBITS;
605         bits |= EXTENT_FIRST_DELALLOC;
606
607         if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
608                 clear = 1;
609 again:
610         if (!prealloc && gfpflags_allow_blocking(mask)) {
611                 /*
612                  * Don't care for allocation failure here because we might end
613                  * up not needing the pre-allocated extent state at all, which
614                  * is the case if we only have in the tree extent states that
615                  * cover our input range and don't cover too any other range.
616                  * If we end up needing a new extent state we allocate it later.
617                  */
618                 prealloc = alloc_extent_state(mask);
619         }
620
621         spin_lock(&tree->lock);
622         if (cached_state) {
623                 cached = *cached_state;
624
625                 if (clear) {
626                         *cached_state = NULL;
627                         cached_state = NULL;
628                 }
629
630                 if (cached && extent_state_in_tree(cached) &&
631                     cached->start <= start && cached->end > start) {
632                         if (clear)
633                                 refcount_dec(&cached->refs);
634                         state = cached;
635                         goto hit_next;
636                 }
637                 if (clear)
638                         free_extent_state(cached);
639         }
640         /*
641          * this search will find the extents that end after
642          * our range starts
643          */
644         node = tree_search(tree, start);
645         if (!node)
646                 goto out;
647         state = rb_entry(node, struct extent_state, rb_node);
648 hit_next:
649         if (state->start > end)
650                 goto out;
651         WARN_ON(state->end < start);
652         last_end = state->end;
653
654         /* the state doesn't have the wanted bits, go ahead */
655         if (!(state->state & bits)) {
656                 state = next_state(state);
657                 goto next;
658         }
659
660         /*
661          *     | ---- desired range ---- |
662          *  | state | or
663          *  | ------------- state -------------- |
664          *
665          * We need to split the extent we found, and may flip
666          * bits on second half.
667          *
668          * If the extent we found extends past our range, we
669          * just split and search again.  It'll get split again
670          * the next time though.
671          *
672          * If the extent we found is inside our range, we clear
673          * the desired bit on it.
674          */
675
676         if (state->start < start) {
677                 prealloc = alloc_extent_state_atomic(prealloc);
678                 BUG_ON(!prealloc);
679                 err = split_state(tree, state, prealloc, start);
680                 if (err)
681                         extent_io_tree_panic(tree, err);
682
683                 prealloc = NULL;
684                 if (err)
685                         goto out;
686                 if (state->end <= end) {
687                         state = clear_state_bit(tree, state, &bits, wake,
688                                                 changeset);
689                         goto next;
690                 }
691                 goto search_again;
692         }
693         /*
694          * | ---- desired range ---- |
695          *                        | state |
696          * We need to split the extent, and clear the bit
697          * on the first half
698          */
699         if (state->start <= end && state->end > end) {
700                 prealloc = alloc_extent_state_atomic(prealloc);
701                 BUG_ON(!prealloc);
702                 err = split_state(tree, state, prealloc, end + 1);
703                 if (err)
704                         extent_io_tree_panic(tree, err);
705
706                 if (wake)
707                         wake_up(&state->wq);
708
709                 clear_state_bit(tree, prealloc, &bits, wake, changeset);
710
711                 prealloc = NULL;
712                 goto out;
713         }
714
715         state = clear_state_bit(tree, state, &bits, wake, changeset);
716 next:
717         if (last_end == (u64)-1)
718                 goto out;
719         start = last_end + 1;
720         if (start <= end && state && !need_resched())
721                 goto hit_next;
722
723 search_again:
724         if (start > end)
725                 goto out;
726         spin_unlock(&tree->lock);
727         if (gfpflags_allow_blocking(mask))
728                 cond_resched();
729         goto again;
730
731 out:
732         spin_unlock(&tree->lock);
733         if (prealloc)
734                 free_extent_state(prealloc);
735
736         return 0;
737
738 }
739
740 static void wait_on_state(struct extent_io_tree *tree,
741                           struct extent_state *state)
742                 __releases(tree->lock)
743                 __acquires(tree->lock)
744 {
745         DEFINE_WAIT(wait);
746         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
747         spin_unlock(&tree->lock);
748         schedule();
749         spin_lock(&tree->lock);
750         finish_wait(&state->wq, &wait);
751 }
752
753 /*
754  * waits for one or more bits to clear on a range in the state tree.
755  * The range [start, end] is inclusive.
756  * The tree lock is taken by this function
757  */
758 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
759                             unsigned long bits)
760 {
761         struct extent_state *state;
762         struct rb_node *node;
763
764         btrfs_debug_check_extent_io_range(tree, start, end);
765
766         spin_lock(&tree->lock);
767 again:
768         while (1) {
769                 /*
770                  * this search will find all the extents that end after
771                  * our range starts
772                  */
773                 node = tree_search(tree, start);
774 process_node:
775                 if (!node)
776                         break;
777
778                 state = rb_entry(node, struct extent_state, rb_node);
779
780                 if (state->start > end)
781                         goto out;
782
783                 if (state->state & bits) {
784                         start = state->start;
785                         refcount_inc(&state->refs);
786                         wait_on_state(tree, state);
787                         free_extent_state(state);
788                         goto again;
789                 }
790                 start = state->end + 1;
791
792                 if (start > end)
793                         break;
794
795                 if (!cond_resched_lock(&tree->lock)) {
796                         node = rb_next(node);
797                         goto process_node;
798                 }
799         }
800 out:
801         spin_unlock(&tree->lock);
802 }
803
804 static void set_state_bits(struct extent_io_tree *tree,
805                            struct extent_state *state,
806                            unsigned *bits, struct extent_changeset *changeset)
807 {
808         unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
809         int ret;
810
811         set_state_cb(tree, state, bits);
812         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
813                 u64 range = state->end - state->start + 1;
814                 tree->dirty_bytes += range;
815         }
816         ret = add_extent_changeset(state, bits_to_set, changeset, 1);
817         BUG_ON(ret < 0);
818         state->state |= bits_to_set;
819 }
820
821 static void cache_state_if_flags(struct extent_state *state,
822                                  struct extent_state **cached_ptr,
823                                  unsigned flags)
824 {
825         if (cached_ptr && !(*cached_ptr)) {
826                 if (!flags || (state->state & flags)) {
827                         *cached_ptr = state;
828                         refcount_inc(&state->refs);
829                 }
830         }
831 }
832
833 static void cache_state(struct extent_state *state,
834                         struct extent_state **cached_ptr)
835 {
836         return cache_state_if_flags(state, cached_ptr,
837                                     EXTENT_IOBITS | EXTENT_BOUNDARY);
838 }
839
840 /*
841  * set some bits on a range in the tree.  This may require allocations or
842  * sleeping, so the gfp mask is used to indicate what is allowed.
843  *
844  * If any of the exclusive bits are set, this will fail with -EEXIST if some
845  * part of the range already has the desired bits set.  The start of the
846  * existing range is returned in failed_start in this case.
847  *
848  * [start, end] is inclusive This takes the tree lock.
849  */
850
851 static int __must_check
852 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
853                  unsigned bits, unsigned exclusive_bits,
854                  u64 *failed_start, struct extent_state **cached_state,
855                  gfp_t mask, struct extent_changeset *changeset)
856 {
857         struct extent_state *state;
858         struct extent_state *prealloc = NULL;
859         struct rb_node *node;
860         struct rb_node **p;
861         struct rb_node *parent;
862         int err = 0;
863         u64 last_start;
864         u64 last_end;
865
866         btrfs_debug_check_extent_io_range(tree, start, end);
867
868         bits |= EXTENT_FIRST_DELALLOC;
869 again:
870         if (!prealloc && gfpflags_allow_blocking(mask)) {
871                 /*
872                  * Don't care for allocation failure here because we might end
873                  * up not needing the pre-allocated extent state at all, which
874                  * is the case if we only have in the tree extent states that
875                  * cover our input range and don't cover too any other range.
876                  * If we end up needing a new extent state we allocate it later.
877                  */
878                 prealloc = alloc_extent_state(mask);
879         }
880
881         spin_lock(&tree->lock);
882         if (cached_state && *cached_state) {
883                 state = *cached_state;
884                 if (state->start <= start && state->end > start &&
885                     extent_state_in_tree(state)) {
886                         node = &state->rb_node;
887                         goto hit_next;
888                 }
889         }
890         /*
891          * this search will find all the extents that end after
892          * our range starts.
893          */
894         node = tree_search_for_insert(tree, start, &p, &parent);
895         if (!node) {
896                 prealloc = alloc_extent_state_atomic(prealloc);
897                 BUG_ON(!prealloc);
898                 err = insert_state(tree, prealloc, start, end,
899                                    &p, &parent, &bits, changeset);
900                 if (err)
901                         extent_io_tree_panic(tree, err);
902
903                 cache_state(prealloc, cached_state);
904                 prealloc = NULL;
905                 goto out;
906         }
907         state = rb_entry(node, struct extent_state, rb_node);
908 hit_next:
909         last_start = state->start;
910         last_end = state->end;
911
912         /*
913          * | ---- desired range ---- |
914          * | state |
915          *
916          * Just lock what we found and keep going
917          */
918         if (state->start == start && state->end <= end) {
919                 if (state->state & exclusive_bits) {
920                         *failed_start = state->start;
921                         err = -EEXIST;
922                         goto out;
923                 }
924
925                 set_state_bits(tree, state, &bits, changeset);
926                 cache_state(state, cached_state);
927                 merge_state(tree, state);
928                 if (last_end == (u64)-1)
929                         goto out;
930                 start = last_end + 1;
931                 state = next_state(state);
932                 if (start < end && state && state->start == start &&
933                     !need_resched())
934                         goto hit_next;
935                 goto search_again;
936         }
937
938         /*
939          *     | ---- desired range ---- |
940          * | state |
941          *   or
942          * | ------------- state -------------- |
943          *
944          * We need to split the extent we found, and may flip bits on
945          * second half.
946          *
947          * If the extent we found extends past our
948          * range, we just split and search again.  It'll get split
949          * again the next time though.
950          *
951          * If the extent we found is inside our range, we set the
952          * desired bit on it.
953          */
954         if (state->start < start) {
955                 if (state->state & exclusive_bits) {
956                         *failed_start = start;
957                         err = -EEXIST;
958                         goto out;
959                 }
960
961                 prealloc = alloc_extent_state_atomic(prealloc);
962                 BUG_ON(!prealloc);
963                 err = split_state(tree, state, prealloc, start);
964                 if (err)
965                         extent_io_tree_panic(tree, err);
966
967                 prealloc = NULL;
968                 if (err)
969                         goto out;
970                 if (state->end <= end) {
971                         set_state_bits(tree, state, &bits, changeset);
972                         cache_state(state, cached_state);
973                         merge_state(tree, state);
974                         if (last_end == (u64)-1)
975                                 goto out;
976                         start = last_end + 1;
977                         state = next_state(state);
978                         if (start < end && state && state->start == start &&
979                             !need_resched())
980                                 goto hit_next;
981                 }
982                 goto search_again;
983         }
984         /*
985          * | ---- desired range ---- |
986          *     | state | or               | state |
987          *
988          * There's a hole, we need to insert something in it and
989          * ignore the extent we found.
990          */
991         if (state->start > start) {
992                 u64 this_end;
993                 if (end < last_start)
994                         this_end = end;
995                 else
996                         this_end = last_start - 1;
997
998                 prealloc = alloc_extent_state_atomic(prealloc);
999                 BUG_ON(!prealloc);
1000
1001                 /*
1002                  * Avoid to free 'prealloc' if it can be merged with
1003                  * the later extent.
1004                  */
1005                 err = insert_state(tree, prealloc, start, this_end,
1006                                    NULL, NULL, &bits, changeset);
1007                 if (err)
1008                         extent_io_tree_panic(tree, err);
1009
1010                 cache_state(prealloc, cached_state);
1011                 prealloc = NULL;
1012                 start = this_end + 1;
1013                 goto search_again;
1014         }
1015         /*
1016          * | ---- desired range ---- |
1017          *                        | state |
1018          * We need to split the extent, and set the bit
1019          * on the first half
1020          */
1021         if (state->start <= end && state->end > end) {
1022                 if (state->state & exclusive_bits) {
1023                         *failed_start = start;
1024                         err = -EEXIST;
1025                         goto out;
1026                 }
1027
1028                 prealloc = alloc_extent_state_atomic(prealloc);
1029                 BUG_ON(!prealloc);
1030                 err = split_state(tree, state, prealloc, end + 1);
1031                 if (err)
1032                         extent_io_tree_panic(tree, err);
1033
1034                 set_state_bits(tree, prealloc, &bits, changeset);
1035                 cache_state(prealloc, cached_state);
1036                 merge_state(tree, prealloc);
1037                 prealloc = NULL;
1038                 goto out;
1039         }
1040
1041 search_again:
1042         if (start > end)
1043                 goto out;
1044         spin_unlock(&tree->lock);
1045         if (gfpflags_allow_blocking(mask))
1046                 cond_resched();
1047         goto again;
1048
1049 out:
1050         spin_unlock(&tree->lock);
1051         if (prealloc)
1052                 free_extent_state(prealloc);
1053
1054         return err;
1055
1056 }
1057
1058 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1059                    unsigned bits, u64 * failed_start,
1060                    struct extent_state **cached_state, gfp_t mask)
1061 {
1062         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1063                                 cached_state, mask, NULL);
1064 }
1065
1066
1067 /**
1068  * convert_extent_bit - convert all bits in a given range from one bit to
1069  *                      another
1070  * @tree:       the io tree to search
1071  * @start:      the start offset in bytes
1072  * @end:        the end offset in bytes (inclusive)
1073  * @bits:       the bits to set in this range
1074  * @clear_bits: the bits to clear in this range
1075  * @cached_state:       state that we're going to cache
1076  *
1077  * This will go through and set bits for the given range.  If any states exist
1078  * already in this range they are set with the given bit and cleared of the
1079  * clear_bits.  This is only meant to be used by things that are mergeable, ie
1080  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1081  * boundary bits like LOCK.
1082  *
1083  * All allocations are done with GFP_NOFS.
1084  */
1085 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1086                        unsigned bits, unsigned clear_bits,
1087                        struct extent_state **cached_state)
1088 {
1089         struct extent_state *state;
1090         struct extent_state *prealloc = NULL;
1091         struct rb_node *node;
1092         struct rb_node **p;
1093         struct rb_node *parent;
1094         int err = 0;
1095         u64 last_start;
1096         u64 last_end;
1097         bool first_iteration = true;
1098
1099         btrfs_debug_check_extent_io_range(tree, start, end);
1100
1101 again:
1102         if (!prealloc) {
1103                 /*
1104                  * Best effort, don't worry if extent state allocation fails
1105                  * here for the first iteration. We might have a cached state
1106                  * that matches exactly the target range, in which case no
1107                  * extent state allocations are needed. We'll only know this
1108                  * after locking the tree.
1109                  */
1110                 prealloc = alloc_extent_state(GFP_NOFS);
1111                 if (!prealloc && !first_iteration)
1112                         return -ENOMEM;
1113         }
1114
1115         spin_lock(&tree->lock);
1116         if (cached_state && *cached_state) {
1117                 state = *cached_state;
1118                 if (state->start <= start && state->end > start &&
1119                     extent_state_in_tree(state)) {
1120                         node = &state->rb_node;
1121                         goto hit_next;
1122                 }
1123         }
1124
1125         /*
1126          * this search will find all the extents that end after
1127          * our range starts.
1128          */
1129         node = tree_search_for_insert(tree, start, &p, &parent);
1130         if (!node) {
1131                 prealloc = alloc_extent_state_atomic(prealloc);
1132                 if (!prealloc) {
1133                         err = -ENOMEM;
1134                         goto out;
1135                 }
1136                 err = insert_state(tree, prealloc, start, end,
1137                                    &p, &parent, &bits, NULL);
1138                 if (err)
1139                         extent_io_tree_panic(tree, err);
1140                 cache_state(prealloc, cached_state);
1141                 prealloc = NULL;
1142                 goto out;
1143         }
1144         state = rb_entry(node, struct extent_state, rb_node);
1145 hit_next:
1146         last_start = state->start;
1147         last_end = state->end;
1148
1149         /*
1150          * | ---- desired range ---- |
1151          * | state |
1152          *
1153          * Just lock what we found and keep going
1154          */
1155         if (state->start == start && state->end <= end) {
1156                 set_state_bits(tree, state, &bits, NULL);
1157                 cache_state(state, cached_state);
1158                 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1159                 if (last_end == (u64)-1)
1160                         goto out;
1161                 start = last_end + 1;
1162                 if (start < end && state && state->start == start &&
1163                     !need_resched())
1164                         goto hit_next;
1165                 goto search_again;
1166         }
1167
1168         /*
1169          *     | ---- desired range ---- |
1170          * | state |
1171          *   or
1172          * | ------------- state -------------- |
1173          *
1174          * We need to split the extent we found, and may flip bits on
1175          * second half.
1176          *
1177          * If the extent we found extends past our
1178          * range, we just split and search again.  It'll get split
1179          * again the next time though.
1180          *
1181          * If the extent we found is inside our range, we set the
1182          * desired bit on it.
1183          */
1184         if (state->start < start) {
1185                 prealloc = alloc_extent_state_atomic(prealloc);
1186                 if (!prealloc) {
1187                         err = -ENOMEM;
1188                         goto out;
1189                 }
1190                 err = split_state(tree, state, prealloc, start);
1191                 if (err)
1192                         extent_io_tree_panic(tree, err);
1193                 prealloc = NULL;
1194                 if (err)
1195                         goto out;
1196                 if (state->end <= end) {
1197                         set_state_bits(tree, state, &bits, NULL);
1198                         cache_state(state, cached_state);
1199                         state = clear_state_bit(tree, state, &clear_bits, 0,
1200                                                 NULL);
1201                         if (last_end == (u64)-1)
1202                                 goto out;
1203                         start = last_end + 1;
1204                         if (start < end && state && state->start == start &&
1205                             !need_resched())
1206                                 goto hit_next;
1207                 }
1208                 goto search_again;
1209         }
1210         /*
1211          * | ---- desired range ---- |
1212          *     | state | or               | state |
1213          *
1214          * There's a hole, we need to insert something in it and
1215          * ignore the extent we found.
1216          */
1217         if (state->start > start) {
1218                 u64 this_end;
1219                 if (end < last_start)
1220                         this_end = end;
1221                 else
1222                         this_end = last_start - 1;
1223
1224                 prealloc = alloc_extent_state_atomic(prealloc);
1225                 if (!prealloc) {
1226                         err = -ENOMEM;
1227                         goto out;
1228                 }
1229
1230                 /*
1231                  * Avoid to free 'prealloc' if it can be merged with
1232                  * the later extent.
1233                  */
1234                 err = insert_state(tree, prealloc, start, this_end,
1235                                    NULL, NULL, &bits, NULL);
1236                 if (err)
1237                         extent_io_tree_panic(tree, err);
1238                 cache_state(prealloc, cached_state);
1239                 prealloc = NULL;
1240                 start = this_end + 1;
1241                 goto search_again;
1242         }
1243         /*
1244          * | ---- desired range ---- |
1245          *                        | state |
1246          * We need to split the extent, and set the bit
1247          * on the first half
1248          */
1249         if (state->start <= end && state->end > end) {
1250                 prealloc = alloc_extent_state_atomic(prealloc);
1251                 if (!prealloc) {
1252                         err = -ENOMEM;
1253                         goto out;
1254                 }
1255
1256                 err = split_state(tree, state, prealloc, end + 1);
1257                 if (err)
1258                         extent_io_tree_panic(tree, err);
1259
1260                 set_state_bits(tree, prealloc, &bits, NULL);
1261                 cache_state(prealloc, cached_state);
1262                 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1263                 prealloc = NULL;
1264                 goto out;
1265         }
1266
1267 search_again:
1268         if (start > end)
1269                 goto out;
1270         spin_unlock(&tree->lock);
1271         cond_resched();
1272         first_iteration = false;
1273         goto again;
1274
1275 out:
1276         spin_unlock(&tree->lock);
1277         if (prealloc)
1278                 free_extent_state(prealloc);
1279
1280         return err;
1281 }
1282
1283 /* wrappers around set/clear extent bit */
1284 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1285                            unsigned bits, struct extent_changeset *changeset)
1286 {
1287         /*
1288          * We don't support EXTENT_LOCKED yet, as current changeset will
1289          * record any bits changed, so for EXTENT_LOCKED case, it will
1290          * either fail with -EEXIST or changeset will record the whole
1291          * range.
1292          */
1293         BUG_ON(bits & EXTENT_LOCKED);
1294
1295         return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1296                                 changeset);
1297 }
1298
1299 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1300                      unsigned bits, int wake, int delete,
1301                      struct extent_state **cached)
1302 {
1303         return __clear_extent_bit(tree, start, end, bits, wake, delete,
1304                                   cached, GFP_NOFS, NULL);
1305 }
1306
1307 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1308                 unsigned bits, struct extent_changeset *changeset)
1309 {
1310         /*
1311          * Don't support EXTENT_LOCKED case, same reason as
1312          * set_record_extent_bits().
1313          */
1314         BUG_ON(bits & EXTENT_LOCKED);
1315
1316         return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1317                                   changeset);
1318 }
1319
1320 /*
1321  * either insert or lock state struct between start and end use mask to tell
1322  * us if waiting is desired.
1323  */
1324 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1325                      struct extent_state **cached_state)
1326 {
1327         int err;
1328         u64 failed_start;
1329
1330         while (1) {
1331                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1332                                        EXTENT_LOCKED, &failed_start,
1333                                        cached_state, GFP_NOFS, NULL);
1334                 if (err == -EEXIST) {
1335                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1336                         start = failed_start;
1337                 } else
1338                         break;
1339                 WARN_ON(start > end);
1340         }
1341         return err;
1342 }
1343
1344 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1345 {
1346         int err;
1347         u64 failed_start;
1348
1349         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1350                                &failed_start, NULL, GFP_NOFS, NULL);
1351         if (err == -EEXIST) {
1352                 if (failed_start > start)
1353                         clear_extent_bit(tree, start, failed_start - 1,
1354                                          EXTENT_LOCKED, 1, 0, NULL);
1355                 return 0;
1356         }
1357         return 1;
1358 }
1359
1360 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1361 {
1362         unsigned long index = start >> PAGE_SHIFT;
1363         unsigned long end_index = end >> PAGE_SHIFT;
1364         struct page *page;
1365
1366         while (index <= end_index) {
1367                 page = find_get_page(inode->i_mapping, index);
1368                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1369                 clear_page_dirty_for_io(page);
1370                 put_page(page);
1371                 index++;
1372         }
1373 }
1374
1375 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1376 {
1377         unsigned long index = start >> PAGE_SHIFT;
1378         unsigned long end_index = end >> PAGE_SHIFT;
1379         struct page *page;
1380
1381         while (index <= end_index) {
1382                 page = find_get_page(inode->i_mapping, index);
1383                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1384                 __set_page_dirty_nobuffers(page);
1385                 account_page_redirty(page);
1386                 put_page(page);
1387                 index++;
1388         }
1389 }
1390
1391 /*
1392  * helper function to set both pages and extents in the tree writeback
1393  */
1394 static void set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1395 {
1396         tree->ops->set_range_writeback(tree->private_data, start, end);
1397 }
1398
1399 /* find the first state struct with 'bits' set after 'start', and
1400  * return it.  tree->lock must be held.  NULL will returned if
1401  * nothing was found after 'start'
1402  */
1403 static struct extent_state *
1404 find_first_extent_bit_state(struct extent_io_tree *tree,
1405                             u64 start, unsigned bits)
1406 {
1407         struct rb_node *node;
1408         struct extent_state *state;
1409
1410         /*
1411          * this search will find all the extents that end after
1412          * our range starts.
1413          */
1414         node = tree_search(tree, start);
1415         if (!node)
1416                 goto out;
1417
1418         while (1) {
1419                 state = rb_entry(node, struct extent_state, rb_node);
1420                 if (state->end >= start && (state->state & bits))
1421                         return state;
1422
1423                 node = rb_next(node);
1424                 if (!node)
1425                         break;
1426         }
1427 out:
1428         return NULL;
1429 }
1430
1431 /*
1432  * find the first offset in the io tree with 'bits' set. zero is
1433  * returned if we find something, and *start_ret and *end_ret are
1434  * set to reflect the state struct that was found.
1435  *
1436  * If nothing was found, 1 is returned. If found something, return 0.
1437  */
1438 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1439                           u64 *start_ret, u64 *end_ret, unsigned bits,
1440                           struct extent_state **cached_state)
1441 {
1442         struct extent_state *state;
1443         struct rb_node *n;
1444         int ret = 1;
1445
1446         spin_lock(&tree->lock);
1447         if (cached_state && *cached_state) {
1448                 state = *cached_state;
1449                 if (state->end == start - 1 && extent_state_in_tree(state)) {
1450                         n = rb_next(&state->rb_node);
1451                         while (n) {
1452                                 state = rb_entry(n, struct extent_state,
1453                                                  rb_node);
1454                                 if (state->state & bits)
1455                                         goto got_it;
1456                                 n = rb_next(n);
1457                         }
1458                         free_extent_state(*cached_state);
1459                         *cached_state = NULL;
1460                         goto out;
1461                 }
1462                 free_extent_state(*cached_state);
1463                 *cached_state = NULL;
1464         }
1465
1466         state = find_first_extent_bit_state(tree, start, bits);
1467 got_it:
1468         if (state) {
1469                 cache_state_if_flags(state, cached_state, 0);
1470                 *start_ret = state->start;
1471                 *end_ret = state->end;
1472                 ret = 0;
1473         }
1474 out:
1475         spin_unlock(&tree->lock);
1476         return ret;
1477 }
1478
1479 /*
1480  * find a contiguous range of bytes in the file marked as delalloc, not
1481  * more than 'max_bytes'.  start and end are used to return the range,
1482  *
1483  * 1 is returned if we find something, 0 if nothing was in the tree
1484  */
1485 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1486                                         u64 *start, u64 *end, u64 max_bytes,
1487                                         struct extent_state **cached_state)
1488 {
1489         struct rb_node *node;
1490         struct extent_state *state;
1491         u64 cur_start = *start;
1492         u64 found = 0;
1493         u64 total_bytes = 0;
1494
1495         spin_lock(&tree->lock);
1496
1497         /*
1498          * this search will find all the extents that end after
1499          * our range starts.
1500          */
1501         node = tree_search(tree, cur_start);
1502         if (!node) {
1503                 if (!found)
1504                         *end = (u64)-1;
1505                 goto out;
1506         }
1507
1508         while (1) {
1509                 state = rb_entry(node, struct extent_state, rb_node);
1510                 if (found && (state->start != cur_start ||
1511                               (state->state & EXTENT_BOUNDARY))) {
1512                         goto out;
1513                 }
1514                 if (!(state->state & EXTENT_DELALLOC)) {
1515                         if (!found)
1516                                 *end = state->end;
1517                         goto out;
1518                 }
1519                 if (!found) {
1520                         *start = state->start;
1521                         *cached_state = state;
1522                         refcount_inc(&state->refs);
1523                 }
1524                 found++;
1525                 *end = state->end;
1526                 cur_start = state->end + 1;
1527                 node = rb_next(node);
1528                 total_bytes += state->end - state->start + 1;
1529                 if (total_bytes >= max_bytes)
1530                         break;
1531                 if (!node)
1532                         break;
1533         }
1534 out:
1535         spin_unlock(&tree->lock);
1536         return found;
1537 }
1538
1539 static int __process_pages_contig(struct address_space *mapping,
1540                                   struct page *locked_page,
1541                                   pgoff_t start_index, pgoff_t end_index,
1542                                   unsigned long page_ops, pgoff_t *index_ret);
1543
1544 static noinline void __unlock_for_delalloc(struct inode *inode,
1545                                            struct page *locked_page,
1546                                            u64 start, u64 end)
1547 {
1548         unsigned long index = start >> PAGE_SHIFT;
1549         unsigned long end_index = end >> PAGE_SHIFT;
1550
1551         ASSERT(locked_page);
1552         if (index == locked_page->index && end_index == index)
1553                 return;
1554
1555         __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1556                                PAGE_UNLOCK, NULL);
1557 }
1558
1559 static noinline int lock_delalloc_pages(struct inode *inode,
1560                                         struct page *locked_page,
1561                                         u64 delalloc_start,
1562                                         u64 delalloc_end)
1563 {
1564         unsigned long index = delalloc_start >> PAGE_SHIFT;
1565         unsigned long index_ret = index;
1566         unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1567         int ret;
1568
1569         ASSERT(locked_page);
1570         if (index == locked_page->index && index == end_index)
1571                 return 0;
1572
1573         ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1574                                      end_index, PAGE_LOCK, &index_ret);
1575         if (ret == -EAGAIN)
1576                 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1577                                       (u64)index_ret << PAGE_SHIFT);
1578         return ret;
1579 }
1580
1581 /*
1582  * find a contiguous range of bytes in the file marked as delalloc, not
1583  * more than 'max_bytes'.  start and end are used to return the range,
1584  *
1585  * 1 is returned if we find something, 0 if nothing was in the tree
1586  */
1587 STATIC u64 find_lock_delalloc_range(struct inode *inode,
1588                                     struct extent_io_tree *tree,
1589                                     struct page *locked_page, u64 *start,
1590                                     u64 *end, u64 max_bytes)
1591 {
1592         u64 delalloc_start;
1593         u64 delalloc_end;
1594         u64 found;
1595         struct extent_state *cached_state = NULL;
1596         int ret;
1597         int loops = 0;
1598
1599 again:
1600         /* step one, find a bunch of delalloc bytes starting at start */
1601         delalloc_start = *start;
1602         delalloc_end = 0;
1603         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1604                                     max_bytes, &cached_state);
1605         if (!found || delalloc_end <= *start) {
1606                 *start = delalloc_start;
1607                 *end = delalloc_end;
1608                 free_extent_state(cached_state);
1609                 return 0;
1610         }
1611
1612         /*
1613          * start comes from the offset of locked_page.  We have to lock
1614          * pages in order, so we can't process delalloc bytes before
1615          * locked_page
1616          */
1617         if (delalloc_start < *start)
1618                 delalloc_start = *start;
1619
1620         /*
1621          * make sure to limit the number of pages we try to lock down
1622          */
1623         if (delalloc_end + 1 - delalloc_start > max_bytes)
1624                 delalloc_end = delalloc_start + max_bytes - 1;
1625
1626         /* step two, lock all the pages after the page that has start */
1627         ret = lock_delalloc_pages(inode, locked_page,
1628                                   delalloc_start, delalloc_end);
1629         if (ret == -EAGAIN) {
1630                 /* some of the pages are gone, lets avoid looping by
1631                  * shortening the size of the delalloc range we're searching
1632                  */
1633                 free_extent_state(cached_state);
1634                 cached_state = NULL;
1635                 if (!loops) {
1636                         max_bytes = PAGE_SIZE;
1637                         loops = 1;
1638                         goto again;
1639                 } else {
1640                         found = 0;
1641                         goto out_failed;
1642                 }
1643         }
1644         BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1645
1646         /* step three, lock the state bits for the whole range */
1647         lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1648
1649         /* then test to make sure it is all still delalloc */
1650         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1651                              EXTENT_DELALLOC, 1, cached_state);
1652         if (!ret) {
1653                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1654                                      &cached_state);
1655                 __unlock_for_delalloc(inode, locked_page,
1656                               delalloc_start, delalloc_end);
1657                 cond_resched();
1658                 goto again;
1659         }
1660         free_extent_state(cached_state);
1661         *start = delalloc_start;
1662         *end = delalloc_end;
1663 out_failed:
1664         return found;
1665 }
1666
1667 static int __process_pages_contig(struct address_space *mapping,
1668                                   struct page *locked_page,
1669                                   pgoff_t start_index, pgoff_t end_index,
1670                                   unsigned long page_ops, pgoff_t *index_ret)
1671 {
1672         unsigned long nr_pages = end_index - start_index + 1;
1673         unsigned long pages_locked = 0;
1674         pgoff_t index = start_index;
1675         struct page *pages[16];
1676         unsigned ret;
1677         int err = 0;
1678         int i;
1679
1680         if (page_ops & PAGE_LOCK) {
1681                 ASSERT(page_ops == PAGE_LOCK);
1682                 ASSERT(index_ret && *index_ret == start_index);
1683         }
1684
1685         if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1686                 mapping_set_error(mapping, -EIO);
1687
1688         while (nr_pages > 0) {
1689                 ret = find_get_pages_contig(mapping, index,
1690                                      min_t(unsigned long,
1691                                      nr_pages, ARRAY_SIZE(pages)), pages);
1692                 if (ret == 0) {
1693                         /*
1694                          * Only if we're going to lock these pages,
1695                          * can we find nothing at @index.
1696                          */
1697                         ASSERT(page_ops & PAGE_LOCK);
1698                         err = -EAGAIN;
1699                         goto out;
1700                 }
1701
1702                 for (i = 0; i < ret; i++) {
1703                         if (page_ops & PAGE_SET_PRIVATE2)
1704                                 SetPagePrivate2(pages[i]);
1705
1706                         if (pages[i] == locked_page) {
1707                                 put_page(pages[i]);
1708                                 pages_locked++;
1709                                 continue;
1710                         }
1711                         if (page_ops & PAGE_CLEAR_DIRTY)
1712                                 clear_page_dirty_for_io(pages[i]);
1713                         if (page_ops & PAGE_SET_WRITEBACK)
1714                                 set_page_writeback(pages[i]);
1715                         if (page_ops & PAGE_SET_ERROR)
1716                                 SetPageError(pages[i]);
1717                         if (page_ops & PAGE_END_WRITEBACK)
1718                                 end_page_writeback(pages[i]);
1719                         if (page_ops & PAGE_UNLOCK)
1720                                 unlock_page(pages[i]);
1721                         if (page_ops & PAGE_LOCK) {
1722                                 lock_page(pages[i]);
1723                                 if (!PageDirty(pages[i]) ||
1724                                     pages[i]->mapping != mapping) {
1725                                         unlock_page(pages[i]);
1726                                         put_page(pages[i]);
1727                                         err = -EAGAIN;
1728                                         goto out;
1729                                 }
1730                         }
1731                         put_page(pages[i]);
1732                         pages_locked++;
1733                 }
1734                 nr_pages -= ret;
1735                 index += ret;
1736                 cond_resched();
1737         }
1738 out:
1739         if (err && index_ret)
1740                 *index_ret = start_index + pages_locked - 1;
1741         return err;
1742 }
1743
1744 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1745                                  u64 delalloc_end, struct page *locked_page,
1746                                  unsigned clear_bits,
1747                                  unsigned long page_ops)
1748 {
1749         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
1750                          NULL);
1751
1752         __process_pages_contig(inode->i_mapping, locked_page,
1753                                start >> PAGE_SHIFT, end >> PAGE_SHIFT,
1754                                page_ops, NULL);
1755 }
1756
1757 /*
1758  * count the number of bytes in the tree that have a given bit(s)
1759  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1760  * cached.  The total number found is returned.
1761  */
1762 u64 count_range_bits(struct extent_io_tree *tree,
1763                      u64 *start, u64 search_end, u64 max_bytes,
1764                      unsigned bits, int contig)
1765 {
1766         struct rb_node *node;
1767         struct extent_state *state;
1768         u64 cur_start = *start;
1769         u64 total_bytes = 0;
1770         u64 last = 0;
1771         int found = 0;
1772
1773         if (WARN_ON(search_end <= cur_start))
1774                 return 0;
1775
1776         spin_lock(&tree->lock);
1777         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1778                 total_bytes = tree->dirty_bytes;
1779                 goto out;
1780         }
1781         /*
1782          * this search will find all the extents that end after
1783          * our range starts.
1784          */
1785         node = tree_search(tree, cur_start);
1786         if (!node)
1787                 goto out;
1788
1789         while (1) {
1790                 state = rb_entry(node, struct extent_state, rb_node);
1791                 if (state->start > search_end)
1792                         break;
1793                 if (contig && found && state->start > last + 1)
1794                         break;
1795                 if (state->end >= cur_start && (state->state & bits) == bits) {
1796                         total_bytes += min(search_end, state->end) + 1 -
1797                                        max(cur_start, state->start);
1798                         if (total_bytes >= max_bytes)
1799                                 break;
1800                         if (!found) {
1801                                 *start = max(cur_start, state->start);
1802                                 found = 1;
1803                         }
1804                         last = state->end;
1805                 } else if (contig && found) {
1806                         break;
1807                 }
1808                 node = rb_next(node);
1809                 if (!node)
1810                         break;
1811         }
1812 out:
1813         spin_unlock(&tree->lock);
1814         return total_bytes;
1815 }
1816
1817 /*
1818  * set the private field for a given byte offset in the tree.  If there isn't
1819  * an extent_state there already, this does nothing.
1820  */
1821 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
1822                 struct io_failure_record *failrec)
1823 {
1824         struct rb_node *node;
1825         struct extent_state *state;
1826         int ret = 0;
1827
1828         spin_lock(&tree->lock);
1829         /*
1830          * this search will find all the extents that end after
1831          * our range starts.
1832          */
1833         node = tree_search(tree, start);
1834         if (!node) {
1835                 ret = -ENOENT;
1836                 goto out;
1837         }
1838         state = rb_entry(node, struct extent_state, rb_node);
1839         if (state->start != start) {
1840                 ret = -ENOENT;
1841                 goto out;
1842         }
1843         state->failrec = failrec;
1844 out:
1845         spin_unlock(&tree->lock);
1846         return ret;
1847 }
1848
1849 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
1850                 struct io_failure_record **failrec)
1851 {
1852         struct rb_node *node;
1853         struct extent_state *state;
1854         int ret = 0;
1855
1856         spin_lock(&tree->lock);
1857         /*
1858          * this search will find all the extents that end after
1859          * our range starts.
1860          */
1861         node = tree_search(tree, start);
1862         if (!node) {
1863                 ret = -ENOENT;
1864                 goto out;
1865         }
1866         state = rb_entry(node, struct extent_state, rb_node);
1867         if (state->start != start) {
1868                 ret = -ENOENT;
1869                 goto out;
1870         }
1871         *failrec = state->failrec;
1872 out:
1873         spin_unlock(&tree->lock);
1874         return ret;
1875 }
1876
1877 /*
1878  * searches a range in the state tree for a given mask.
1879  * If 'filled' == 1, this returns 1 only if every extent in the tree
1880  * has the bits set.  Otherwise, 1 is returned if any bit in the
1881  * range is found set.
1882  */
1883 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1884                    unsigned bits, int filled, struct extent_state *cached)
1885 {
1886         struct extent_state *state = NULL;
1887         struct rb_node *node;
1888         int bitset = 0;
1889
1890         spin_lock(&tree->lock);
1891         if (cached && extent_state_in_tree(cached) && cached->start <= start &&
1892             cached->end > start)
1893                 node = &cached->rb_node;
1894         else
1895                 node = tree_search(tree, start);
1896         while (node && start <= end) {
1897                 state = rb_entry(node, struct extent_state, rb_node);
1898
1899                 if (filled && state->start > start) {
1900                         bitset = 0;
1901                         break;
1902                 }
1903
1904                 if (state->start > end)
1905                         break;
1906
1907                 if (state->state & bits) {
1908                         bitset = 1;
1909                         if (!filled)
1910                                 break;
1911                 } else if (filled) {
1912                         bitset = 0;
1913                         break;
1914                 }
1915
1916                 if (state->end == (u64)-1)
1917                         break;
1918
1919                 start = state->end + 1;
1920                 if (start > end)
1921                         break;
1922                 node = rb_next(node);
1923                 if (!node) {
1924                         if (filled)
1925                                 bitset = 0;
1926                         break;
1927                 }
1928         }
1929         spin_unlock(&tree->lock);
1930         return bitset;
1931 }
1932
1933 /*
1934  * helper function to set a given page up to date if all the
1935  * extents in the tree for that page are up to date
1936  */
1937 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1938 {
1939         u64 start = page_offset(page);
1940         u64 end = start + PAGE_SIZE - 1;
1941         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1942                 SetPageUptodate(page);
1943 }
1944
1945 int free_io_failure(struct extent_io_tree *failure_tree,
1946                     struct extent_io_tree *io_tree,
1947                     struct io_failure_record *rec)
1948 {
1949         int ret;
1950         int err = 0;
1951
1952         set_state_failrec(failure_tree, rec->start, NULL);
1953         ret = clear_extent_bits(failure_tree, rec->start,
1954                                 rec->start + rec->len - 1,
1955                                 EXTENT_LOCKED | EXTENT_DIRTY);
1956         if (ret)
1957                 err = ret;
1958
1959         ret = clear_extent_bits(io_tree, rec->start,
1960                                 rec->start + rec->len - 1,
1961                                 EXTENT_DAMAGED);
1962         if (ret && !err)
1963                 err = ret;
1964
1965         kfree(rec);
1966         return err;
1967 }
1968
1969 /*
1970  * this bypasses the standard btrfs submit functions deliberately, as
1971  * the standard behavior is to write all copies in a raid setup. here we only
1972  * want to write the one bad copy. so we do the mapping for ourselves and issue
1973  * submit_bio directly.
1974  * to avoid any synchronization issues, wait for the data after writing, which
1975  * actually prevents the read that triggered the error from finishing.
1976  * currently, there can be no more than two copies of every data bit. thus,
1977  * exactly one rewrite is required.
1978  */
1979 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1980                       u64 length, u64 logical, struct page *page,
1981                       unsigned int pg_offset, int mirror_num)
1982 {
1983         struct bio *bio;
1984         struct btrfs_device *dev;
1985         u64 map_length = 0;
1986         u64 sector;
1987         struct btrfs_bio *bbio = NULL;
1988         int ret;
1989
1990         ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
1991         BUG_ON(!mirror_num);
1992
1993         bio = btrfs_io_bio_alloc(1);
1994         bio->bi_iter.bi_size = 0;
1995         map_length = length;
1996
1997         /*
1998          * Avoid races with device replace and make sure our bbio has devices
1999          * associated to its stripes that don't go away while we are doing the
2000          * read repair operation.
2001          */
2002         btrfs_bio_counter_inc_blocked(fs_info);
2003         if (btrfs_is_parity_mirror(fs_info, logical, length)) {
2004                 /*
2005                  * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2006                  * to update all raid stripes, but here we just want to correct
2007                  * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2008                  * stripe's dev and sector.
2009                  */
2010                 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2011                                       &map_length, &bbio, 0);
2012                 if (ret) {
2013                         btrfs_bio_counter_dec(fs_info);
2014                         bio_put(bio);
2015                         return -EIO;
2016                 }
2017                 ASSERT(bbio->mirror_num == 1);
2018         } else {
2019                 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2020                                       &map_length, &bbio, mirror_num);
2021                 if (ret) {
2022                         btrfs_bio_counter_dec(fs_info);
2023                         bio_put(bio);
2024                         return -EIO;
2025                 }
2026                 BUG_ON(mirror_num != bbio->mirror_num);
2027         }
2028
2029         sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2030         bio->bi_iter.bi_sector = sector;
2031         dev = bbio->stripes[bbio->mirror_num - 1].dev;
2032         btrfs_put_bbio(bbio);
2033         if (!dev || !dev->bdev ||
2034             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2035                 btrfs_bio_counter_dec(fs_info);
2036                 bio_put(bio);
2037                 return -EIO;
2038         }
2039         bio_set_dev(bio, dev->bdev);
2040         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2041         bio_add_page(bio, page, length, pg_offset);
2042
2043         if (btrfsic_submit_bio_wait(bio)) {
2044                 /* try to remap that extent elsewhere? */
2045                 btrfs_bio_counter_dec(fs_info);
2046                 bio_put(bio);
2047                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2048                 return -EIO;
2049         }
2050
2051         btrfs_info_rl_in_rcu(fs_info,
2052                 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2053                                   ino, start,
2054                                   rcu_str_deref(dev->name), sector);
2055         btrfs_bio_counter_dec(fs_info);
2056         bio_put(bio);
2057         return 0;
2058 }
2059
2060 int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2061                          struct extent_buffer *eb, int mirror_num)
2062 {
2063         u64 start = eb->start;
2064         unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
2065         int ret = 0;
2066
2067         if (sb_rdonly(fs_info->sb))
2068                 return -EROFS;
2069
2070         for (i = 0; i < num_pages; i++) {
2071                 struct page *p = eb->pages[i];
2072
2073                 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2074                                         start - page_offset(p), mirror_num);
2075                 if (ret)
2076                         break;
2077                 start += PAGE_SIZE;
2078         }
2079
2080         return ret;
2081 }
2082
2083 /*
2084  * each time an IO finishes, we do a fast check in the IO failure tree
2085  * to see if we need to process or clean up an io_failure_record
2086  */
2087 int clean_io_failure(struct btrfs_fs_info *fs_info,
2088                      struct extent_io_tree *failure_tree,
2089                      struct extent_io_tree *io_tree, u64 start,
2090                      struct page *page, u64 ino, unsigned int pg_offset)
2091 {
2092         u64 private;
2093         struct io_failure_record *failrec;
2094         struct extent_state *state;
2095         int num_copies;
2096         int ret;
2097
2098         private = 0;
2099         ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2100                                EXTENT_DIRTY, 0);
2101         if (!ret)
2102                 return 0;
2103
2104         ret = get_state_failrec(failure_tree, start, &failrec);
2105         if (ret)
2106                 return 0;
2107
2108         BUG_ON(!failrec->this_mirror);
2109
2110         if (failrec->in_validation) {
2111                 /* there was no real error, just free the record */
2112                 btrfs_debug(fs_info,
2113                         "clean_io_failure: freeing dummy error at %llu",
2114                         failrec->start);
2115                 goto out;
2116         }
2117         if (sb_rdonly(fs_info->sb))
2118                 goto out;
2119
2120         spin_lock(&io_tree->lock);
2121         state = find_first_extent_bit_state(io_tree,
2122                                             failrec->start,
2123                                             EXTENT_LOCKED);
2124         spin_unlock(&io_tree->lock);
2125
2126         if (state && state->start <= failrec->start &&
2127             state->end >= failrec->start + failrec->len - 1) {
2128                 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2129                                               failrec->len);
2130                 if (num_copies > 1)  {
2131                         repair_io_failure(fs_info, ino, start, failrec->len,
2132                                           failrec->logical, page, pg_offset,
2133                                           failrec->failed_mirror);
2134                 }
2135         }
2136
2137 out:
2138         free_io_failure(failure_tree, io_tree, failrec);
2139
2140         return 0;
2141 }
2142
2143 /*
2144  * Can be called when
2145  * - hold extent lock
2146  * - under ordered extent
2147  * - the inode is freeing
2148  */
2149 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2150 {
2151         struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2152         struct io_failure_record *failrec;
2153         struct extent_state *state, *next;
2154
2155         if (RB_EMPTY_ROOT(&failure_tree->state))
2156                 return;
2157
2158         spin_lock(&failure_tree->lock);
2159         state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2160         while (state) {
2161                 if (state->start > end)
2162                         break;
2163
2164                 ASSERT(state->end <= end);
2165
2166                 next = next_state(state);
2167
2168                 failrec = state->failrec;
2169                 free_extent_state(state);
2170                 kfree(failrec);
2171
2172                 state = next;
2173         }
2174         spin_unlock(&failure_tree->lock);
2175 }
2176
2177 int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2178                 struct io_failure_record **failrec_ret)
2179 {
2180         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2181         struct io_failure_record *failrec;
2182         struct extent_map *em;
2183         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2184         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2185         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2186         int ret;
2187         u64 logical;
2188
2189         ret = get_state_failrec(failure_tree, start, &failrec);
2190         if (ret) {
2191                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2192                 if (!failrec)
2193                         return -ENOMEM;
2194
2195                 failrec->start = start;
2196                 failrec->len = end - start + 1;
2197                 failrec->this_mirror = 0;
2198                 failrec->bio_flags = 0;
2199                 failrec->in_validation = 0;
2200
2201                 read_lock(&em_tree->lock);
2202                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2203                 if (!em) {
2204                         read_unlock(&em_tree->lock);
2205                         kfree(failrec);
2206                         return -EIO;
2207                 }
2208
2209                 if (em->start > start || em->start + em->len <= start) {
2210                         free_extent_map(em);
2211                         em = NULL;
2212                 }
2213                 read_unlock(&em_tree->lock);
2214                 if (!em) {
2215                         kfree(failrec);
2216                         return -EIO;
2217                 }
2218
2219                 logical = start - em->start;
2220                 logical = em->block_start + logical;
2221                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2222                         logical = em->block_start;
2223                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2224                         extent_set_compress_type(&failrec->bio_flags,
2225                                                  em->compress_type);
2226                 }
2227
2228                 btrfs_debug(fs_info,
2229                         "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2230                         logical, start, failrec->len);
2231
2232                 failrec->logical = logical;
2233                 free_extent_map(em);
2234
2235                 /* set the bits in the private failure tree */
2236                 ret = set_extent_bits(failure_tree, start, end,
2237                                         EXTENT_LOCKED | EXTENT_DIRTY);
2238                 if (ret >= 0)
2239                         ret = set_state_failrec(failure_tree, start, failrec);
2240                 /* set the bits in the inode's tree */
2241                 if (ret >= 0)
2242                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2243                 if (ret < 0) {
2244                         kfree(failrec);
2245                         return ret;
2246                 }
2247         } else {
2248                 btrfs_debug(fs_info,
2249                         "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2250                         failrec->logical, failrec->start, failrec->len,
2251                         failrec->in_validation);
2252                 /*
2253                  * when data can be on disk more than twice, add to failrec here
2254                  * (e.g. with a list for failed_mirror) to make
2255                  * clean_io_failure() clean all those errors at once.
2256                  */
2257         }
2258
2259         *failrec_ret = failrec;
2260
2261         return 0;
2262 }
2263
2264 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2265                            struct io_failure_record *failrec, int failed_mirror)
2266 {
2267         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2268         int num_copies;
2269
2270         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2271         if (num_copies == 1) {
2272                 /*
2273                  * we only have a single copy of the data, so don't bother with
2274                  * all the retry and error correction code that follows. no
2275                  * matter what the error is, it is very likely to persist.
2276                  */
2277                 btrfs_debug(fs_info,
2278                         "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2279                         num_copies, failrec->this_mirror, failed_mirror);
2280                 return false;
2281         }
2282
2283         /*
2284          * there are two premises:
2285          *      a) deliver good data to the caller
2286          *      b) correct the bad sectors on disk
2287          */
2288         if (failed_bio_pages > 1) {
2289                 /*
2290                  * to fulfill b), we need to know the exact failing sectors, as
2291                  * we don't want to rewrite any more than the failed ones. thus,
2292                  * we need separate read requests for the failed bio
2293                  *
2294                  * if the following BUG_ON triggers, our validation request got
2295                  * merged. we need separate requests for our algorithm to work.
2296                  */
2297                 BUG_ON(failrec->in_validation);
2298                 failrec->in_validation = 1;
2299                 failrec->this_mirror = failed_mirror;
2300         } else {
2301                 /*
2302                  * we're ready to fulfill a) and b) alongside. get a good copy
2303                  * of the failed sector and if we succeed, we have setup
2304                  * everything for repair_io_failure to do the rest for us.
2305                  */
2306                 if (failrec->in_validation) {
2307                         BUG_ON(failrec->this_mirror != failed_mirror);
2308                         failrec->in_validation = 0;
2309                         failrec->this_mirror = 0;
2310                 }
2311                 failrec->failed_mirror = failed_mirror;
2312                 failrec->this_mirror++;
2313                 if (failrec->this_mirror == failed_mirror)
2314                         failrec->this_mirror++;
2315         }
2316
2317         if (failrec->this_mirror > num_copies) {
2318                 btrfs_debug(fs_info,
2319                         "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2320                         num_copies, failrec->this_mirror, failed_mirror);
2321                 return false;
2322         }
2323
2324         return true;
2325 }
2326
2327
2328 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2329                                     struct io_failure_record *failrec,
2330                                     struct page *page, int pg_offset, int icsum,
2331                                     bio_end_io_t *endio_func, void *data)
2332 {
2333         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2334         struct bio *bio;
2335         struct btrfs_io_bio *btrfs_failed_bio;
2336         struct btrfs_io_bio *btrfs_bio;
2337
2338         bio = btrfs_io_bio_alloc(1);
2339         bio->bi_end_io = endio_func;
2340         bio->bi_iter.bi_sector = failrec->logical >> 9;
2341         bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
2342         bio->bi_iter.bi_size = 0;
2343         bio->bi_private = data;
2344
2345         btrfs_failed_bio = btrfs_io_bio(failed_bio);
2346         if (btrfs_failed_bio->csum) {
2347                 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2348
2349                 btrfs_bio = btrfs_io_bio(bio);
2350                 btrfs_bio->csum = btrfs_bio->csum_inline;
2351                 icsum *= csum_size;
2352                 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2353                        csum_size);
2354         }
2355
2356         bio_add_page(bio, page, failrec->len, pg_offset);
2357
2358         return bio;
2359 }
2360
2361 /*
2362  * this is a generic handler for readpage errors (default
2363  * readpage_io_failed_hook). if other copies exist, read those and write back
2364  * good data to the failed position. does not investigate in remapping the
2365  * failed extent elsewhere, hoping the device will be smart enough to do this as
2366  * needed
2367  */
2368
2369 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2370                               struct page *page, u64 start, u64 end,
2371                               int failed_mirror)
2372 {
2373         struct io_failure_record *failrec;
2374         struct inode *inode = page->mapping->host;
2375         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2376         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2377         struct bio *bio;
2378         int read_mode = 0;
2379         blk_status_t status;
2380         int ret;
2381         unsigned failed_bio_pages = bio_pages_all(failed_bio);
2382
2383         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2384
2385         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2386         if (ret)
2387                 return ret;
2388
2389         if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2390                                     failed_mirror)) {
2391                 free_io_failure(failure_tree, tree, failrec);
2392                 return -EIO;
2393         }
2394
2395         if (failed_bio_pages > 1)
2396                 read_mode |= REQ_FAILFAST_DEV;
2397
2398         phy_offset >>= inode->i_sb->s_blocksize_bits;
2399         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2400                                       start - page_offset(page),
2401                                       (int)phy_offset, failed_bio->bi_end_io,
2402                                       NULL);
2403         bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
2404
2405         btrfs_debug(btrfs_sb(inode->i_sb),
2406                 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2407                 read_mode, failrec->this_mirror, failrec->in_validation);
2408
2409         status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2410                                          failrec->bio_flags, 0);
2411         if (status) {
2412                 free_io_failure(failure_tree, tree, failrec);
2413                 bio_put(bio);
2414                 ret = blk_status_to_errno(status);
2415         }
2416
2417         return ret;
2418 }
2419
2420 /* lots and lots of room for performance fixes in the end_bio funcs */
2421
2422 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2423 {
2424         int uptodate = (err == 0);
2425         struct extent_io_tree *tree;
2426         int ret = 0;
2427
2428         tree = &BTRFS_I(page->mapping->host)->io_tree;
2429
2430         if (tree->ops && tree->ops->writepage_end_io_hook)
2431                 tree->ops->writepage_end_io_hook(page, start, end, NULL,
2432                                 uptodate);
2433
2434         if (!uptodate) {
2435                 ClearPageUptodate(page);
2436                 SetPageError(page);
2437                 ret = err < 0 ? err : -EIO;
2438                 mapping_set_error(page->mapping, ret);
2439         }
2440 }
2441
2442 /*
2443  * after a writepage IO is done, we need to:
2444  * clear the uptodate bits on error
2445  * clear the writeback bits in the extent tree for this IO
2446  * end_page_writeback if the page has no more pending IO
2447  *
2448  * Scheduling is not allowed, so the extent state tree is expected
2449  * to have one and only one object corresponding to this IO.
2450  */
2451 static void end_bio_extent_writepage(struct bio *bio)
2452 {
2453         int error = blk_status_to_errno(bio->bi_status);
2454         struct bio_vec *bvec;
2455         u64 start;
2456         u64 end;
2457         int i;
2458
2459         ASSERT(!bio_flagged(bio, BIO_CLONED));
2460         bio_for_each_segment_all(bvec, bio, i) {
2461                 struct page *page = bvec->bv_page;
2462                 struct inode *inode = page->mapping->host;
2463                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2464
2465                 /* We always issue full-page reads, but if some block
2466                  * in a page fails to read, blk_update_request() will
2467                  * advance bv_offset and adjust bv_len to compensate.
2468                  * Print a warning for nonzero offsets, and an error
2469                  * if they don't add up to a full page.  */
2470                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2471                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2472                                 btrfs_err(fs_info,
2473                                    "partial page write in btrfs with offset %u and length %u",
2474                                         bvec->bv_offset, bvec->bv_len);
2475                         else
2476                                 btrfs_info(fs_info,
2477                                    "incomplete page write in btrfs with offset %u and length %u",
2478                                         bvec->bv_offset, bvec->bv_len);
2479                 }
2480
2481                 start = page_offset(page);
2482                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2483
2484                 end_extent_writepage(page, error, start, end);
2485                 end_page_writeback(page);
2486         }
2487
2488         bio_put(bio);
2489 }
2490
2491 static void
2492 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2493                               int uptodate)
2494 {
2495         struct extent_state *cached = NULL;
2496         u64 end = start + len - 1;
2497
2498         if (uptodate && tree->track_uptodate)
2499                 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2500         unlock_extent_cached_atomic(tree, start, end, &cached);
2501 }
2502
2503 /*
2504  * after a readpage IO is done, we need to:
2505  * clear the uptodate bits on error
2506  * set the uptodate bits if things worked
2507  * set the page up to date if all extents in the tree are uptodate
2508  * clear the lock bit in the extent tree
2509  * unlock the page if there are no other extents locked for it
2510  *
2511  * Scheduling is not allowed, so the extent state tree is expected
2512  * to have one and only one object corresponding to this IO.
2513  */
2514 static void end_bio_extent_readpage(struct bio *bio)
2515 {
2516         struct bio_vec *bvec;
2517         int uptodate = !bio->bi_status;
2518         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2519         struct extent_io_tree *tree, *failure_tree;
2520         u64 offset = 0;
2521         u64 start;
2522         u64 end;
2523         u64 len;
2524         u64 extent_start = 0;
2525         u64 extent_len = 0;
2526         int mirror;
2527         int ret;
2528         int i;
2529
2530         ASSERT(!bio_flagged(bio, BIO_CLONED));
2531         bio_for_each_segment_all(bvec, bio, i) {
2532                 struct page *page = bvec->bv_page;
2533                 struct inode *inode = page->mapping->host;
2534                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2535
2536                 btrfs_debug(fs_info,
2537                         "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2538                         (u64)bio->bi_iter.bi_sector, bio->bi_status,
2539                         io_bio->mirror_num);
2540                 tree = &BTRFS_I(inode)->io_tree;
2541                 failure_tree = &BTRFS_I(inode)->io_failure_tree;
2542
2543                 /* We always issue full-page reads, but if some block
2544                  * in a page fails to read, blk_update_request() will
2545                  * advance bv_offset and adjust bv_len to compensate.
2546                  * Print a warning for nonzero offsets, and an error
2547                  * if they don't add up to a full page.  */
2548                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2549                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2550                                 btrfs_err(fs_info,
2551                                         "partial page read in btrfs with offset %u and length %u",
2552                                         bvec->bv_offset, bvec->bv_len);
2553                         else
2554                                 btrfs_info(fs_info,
2555                                         "incomplete page read in btrfs with offset %u and length %u",
2556                                         bvec->bv_offset, bvec->bv_len);
2557                 }
2558
2559                 start = page_offset(page);
2560                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2561                 len = bvec->bv_len;
2562
2563                 mirror = io_bio->mirror_num;
2564                 if (likely(uptodate && tree->ops)) {
2565                         ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2566                                                               page, start, end,
2567                                                               mirror);
2568                         if (ret)
2569                                 uptodate = 0;
2570                         else
2571                                 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2572                                                  failure_tree, tree, start,
2573                                                  page,
2574                                                  btrfs_ino(BTRFS_I(inode)), 0);
2575                 }
2576
2577                 if (likely(uptodate))
2578                         goto readpage_ok;
2579
2580                 if (tree->ops) {
2581                         ret = tree->ops->readpage_io_failed_hook(page, mirror);
2582                         if (ret == -EAGAIN) {
2583                                 /*
2584                                  * Data inode's readpage_io_failed_hook() always
2585                                  * returns -EAGAIN.
2586                                  *
2587                                  * The generic bio_readpage_error handles errors
2588                                  * the following way: If possible, new read
2589                                  * requests are created and submitted and will
2590                                  * end up in end_bio_extent_readpage as well (if
2591                                  * we're lucky, not in the !uptodate case). In
2592                                  * that case it returns 0 and we just go on with
2593                                  * the next page in our bio. If it can't handle
2594                                  * the error it will return -EIO and we remain
2595                                  * responsible for that page.
2596                                  */
2597                                 ret = bio_readpage_error(bio, offset, page,
2598                                                          start, end, mirror);
2599                                 if (ret == 0) {
2600                                         uptodate = !bio->bi_status;
2601                                         offset += len;
2602                                         continue;
2603                                 }
2604                         }
2605
2606                         /*
2607                          * metadata's readpage_io_failed_hook() always returns
2608                          * -EIO and fixes nothing.  -EIO is also returned if
2609                          * data inode error could not be fixed.
2610                          */
2611                         ASSERT(ret == -EIO);
2612                 }
2613 readpage_ok:
2614                 if (likely(uptodate)) {
2615                         loff_t i_size = i_size_read(inode);
2616                         pgoff_t end_index = i_size >> PAGE_SHIFT;
2617                         unsigned off;
2618
2619                         /* Zero out the end if this page straddles i_size */
2620                         off = i_size & (PAGE_SIZE-1);
2621                         if (page->index == end_index && off)
2622                                 zero_user_segment(page, off, PAGE_SIZE);
2623                         SetPageUptodate(page);
2624                 } else {
2625                         ClearPageUptodate(page);
2626                         SetPageError(page);
2627                 }
2628                 unlock_page(page);
2629                 offset += len;
2630
2631                 if (unlikely(!uptodate)) {
2632                         if (extent_len) {
2633                                 endio_readpage_release_extent(tree,
2634                                                               extent_start,
2635                                                               extent_len, 1);
2636                                 extent_start = 0;
2637                                 extent_len = 0;
2638                         }
2639                         endio_readpage_release_extent(tree, start,
2640                                                       end - start + 1, 0);
2641                 } else if (!extent_len) {
2642                         extent_start = start;
2643                         extent_len = end + 1 - start;
2644                 } else if (extent_start + extent_len == start) {
2645                         extent_len += end + 1 - start;
2646                 } else {
2647                         endio_readpage_release_extent(tree, extent_start,
2648                                                       extent_len, uptodate);
2649                         extent_start = start;
2650                         extent_len = end + 1 - start;
2651                 }
2652         }
2653
2654         if (extent_len)
2655                 endio_readpage_release_extent(tree, extent_start, extent_len,
2656                                               uptodate);
2657         if (io_bio->end_io)
2658                 io_bio->end_io(io_bio, blk_status_to_errno(bio->bi_status));
2659         bio_put(bio);
2660 }
2661
2662 /*
2663  * Initialize the members up to but not including 'bio'. Use after allocating a
2664  * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2665  * 'bio' because use of __GFP_ZERO is not supported.
2666  */
2667 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
2668 {
2669         memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2670 }
2671
2672 /*
2673  * The following helpers allocate a bio. As it's backed by a bioset, it'll
2674  * never fail.  We're returning a bio right now but you can call btrfs_io_bio
2675  * for the appropriate container_of magic
2676  */
2677 struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
2678 {
2679         struct bio *bio;
2680
2681         bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, btrfs_bioset);
2682         bio_set_dev(bio, bdev);
2683         bio->bi_iter.bi_sector = first_byte >> 9;
2684         btrfs_io_bio_init(btrfs_io_bio(bio));
2685         return bio;
2686 }
2687
2688 struct bio *btrfs_bio_clone(struct bio *bio)
2689 {
2690         struct btrfs_io_bio *btrfs_bio;
2691         struct bio *new;
2692
2693         /* Bio allocation backed by a bioset does not fail */
2694         new = bio_clone_fast(bio, GFP_NOFS, btrfs_bioset);
2695         btrfs_bio = btrfs_io_bio(new);
2696         btrfs_io_bio_init(btrfs_bio);
2697         btrfs_bio->iter = bio->bi_iter;
2698         return new;
2699 }
2700
2701 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
2702 {
2703         struct bio *bio;
2704
2705         /* Bio allocation backed by a bioset does not fail */
2706         bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, btrfs_bioset);
2707         btrfs_io_bio_init(btrfs_io_bio(bio));
2708         return bio;
2709 }
2710
2711 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
2712 {
2713         struct bio *bio;
2714         struct btrfs_io_bio *btrfs_bio;
2715
2716         /* this will never fail when it's backed by a bioset */
2717         bio = bio_clone_fast(orig, GFP_NOFS, btrfs_bioset);
2718         ASSERT(bio);
2719
2720         btrfs_bio = btrfs_io_bio(bio);
2721         btrfs_io_bio_init(btrfs_bio);
2722
2723         bio_trim(bio, offset >> 9, size >> 9);
2724         btrfs_bio->iter = bio->bi_iter;
2725         return bio;
2726 }
2727
2728 static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
2729                                        unsigned long bio_flags)
2730 {
2731         blk_status_t ret = 0;
2732         struct bio_vec *bvec = bio_last_bvec_all(bio);
2733         struct page *page = bvec->bv_page;
2734         struct extent_io_tree *tree = bio->bi_private;
2735         u64 start;
2736
2737         start = page_offset(page) + bvec->bv_offset;
2738
2739         bio->bi_private = NULL;
2740
2741         if (tree->ops)
2742                 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
2743                                            mirror_num, bio_flags, start);
2744         else
2745                 btrfsic_submit_bio(bio);
2746
2747         return blk_status_to_errno(ret);
2748 }
2749
2750 /*
2751  * @opf:        bio REQ_OP_* and REQ_* flags as one value
2752  * @tree:       tree so we can call our merge_bio hook
2753  * @wbc:        optional writeback control for io accounting
2754  * @page:       page to add to the bio
2755  * @pg_offset:  offset of the new bio or to check whether we are adding
2756  *              a contiguous page to the previous one
2757  * @size:       portion of page that we want to write
2758  * @offset:     starting offset in the page
2759  * @bdev:       attach newly created bios to this bdev
2760  * @bio_ret:    must be valid pointer, newly allocated bio will be stored there
2761  * @end_io_func:     end_io callback for new bio
2762  * @mirror_num:      desired mirror to read/write
2763  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
2764  * @bio_flags:  flags of the current bio to see if we can merge them
2765  */
2766 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2767                               struct writeback_control *wbc,
2768                               struct page *page, u64 offset,
2769                               size_t size, unsigned long pg_offset,
2770                               struct block_device *bdev,
2771                               struct bio **bio_ret,
2772                               bio_end_io_t end_io_func,
2773                               int mirror_num,
2774                               unsigned long prev_bio_flags,
2775                               unsigned long bio_flags,
2776                               bool force_bio_submit)
2777 {
2778         int ret = 0;
2779         struct bio *bio;
2780         size_t page_size = min_t(size_t, size, PAGE_SIZE);
2781         sector_t sector = offset >> 9;
2782
2783         ASSERT(bio_ret);
2784
2785         if (*bio_ret) {
2786                 bool contig;
2787                 bool can_merge = true;
2788
2789                 bio = *bio_ret;
2790                 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2791                         contig = bio->bi_iter.bi_sector == sector;
2792                 else
2793                         contig = bio_end_sector(bio) == sector;
2794
2795                 if (tree->ops && tree->ops->merge_bio_hook(page, offset,
2796                                         page_size, bio, bio_flags))
2797                         can_merge = false;
2798
2799                 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2800                     force_bio_submit ||
2801                     bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2802                         ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2803                         if (ret < 0) {
2804                                 *bio_ret = NULL;
2805                                 return ret;
2806                         }
2807                         bio = NULL;
2808                 } else {
2809                         if (wbc)
2810                                 wbc_account_io(wbc, page, page_size);
2811                         return 0;
2812                 }
2813         }
2814
2815         bio = btrfs_bio_alloc(bdev, offset);
2816         bio_add_page(bio, page, page_size, pg_offset);
2817         bio->bi_end_io = end_io_func;
2818         bio->bi_private = tree;
2819         bio->bi_write_hint = page->mapping->host->i_write_hint;
2820         bio->bi_opf = opf;
2821         if (wbc) {
2822                 wbc_init_bio(wbc, bio);
2823                 wbc_account_io(wbc, page, page_size);
2824         }
2825
2826         *bio_ret = bio;
2827
2828         return ret;
2829 }
2830
2831 static void attach_extent_buffer_page(struct extent_buffer *eb,
2832                                       struct page *page)
2833 {
2834         if (!PagePrivate(page)) {
2835                 SetPagePrivate(page);
2836                 get_page(page);
2837                 set_page_private(page, (unsigned long)eb);
2838         } else {
2839                 WARN_ON(page->private != (unsigned long)eb);
2840         }
2841 }
2842
2843 void set_page_extent_mapped(struct page *page)
2844 {
2845         if (!PagePrivate(page)) {
2846                 SetPagePrivate(page);
2847                 get_page(page);
2848                 set_page_private(page, EXTENT_PAGE_PRIVATE);
2849         }
2850 }
2851
2852 static struct extent_map *
2853 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2854                  u64 start, u64 len, get_extent_t *get_extent,
2855                  struct extent_map **em_cached)
2856 {
2857         struct extent_map *em;
2858
2859         if (em_cached && *em_cached) {
2860                 em = *em_cached;
2861                 if (extent_map_in_tree(em) && start >= em->start &&
2862                     start < extent_map_end(em)) {
2863                         refcount_inc(&em->refs);
2864                         return em;
2865                 }
2866
2867                 free_extent_map(em);
2868                 *em_cached = NULL;
2869         }
2870
2871         em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
2872         if (em_cached && !IS_ERR_OR_NULL(em)) {
2873                 BUG_ON(*em_cached);
2874                 refcount_inc(&em->refs);
2875                 *em_cached = em;
2876         }
2877         return em;
2878 }
2879 /*
2880  * basic readpage implementation.  Locked extent state structs are inserted
2881  * into the tree that are removed when the IO is done (by the end_io
2882  * handlers)
2883  * XXX JDM: This needs looking at to ensure proper page locking
2884  * return 0 on success, otherwise return error
2885  */
2886 static int __do_readpage(struct extent_io_tree *tree,
2887                          struct page *page,
2888                          get_extent_t *get_extent,
2889                          struct extent_map **em_cached,
2890                          struct bio **bio, int mirror_num,
2891                          unsigned long *bio_flags, unsigned int read_flags,
2892                          u64 *prev_em_start)
2893 {
2894         struct inode *inode = page->mapping->host;
2895         u64 start = page_offset(page);
2896         const u64 end = start + PAGE_SIZE - 1;
2897         u64 cur = start;
2898         u64 extent_offset;
2899         u64 last_byte = i_size_read(inode);
2900         u64 block_start;
2901         u64 cur_end;
2902         struct extent_map *em;
2903         struct block_device *bdev;
2904         int ret = 0;
2905         int nr = 0;
2906         size_t pg_offset = 0;
2907         size_t iosize;
2908         size_t disk_io_size;
2909         size_t blocksize = inode->i_sb->s_blocksize;
2910         unsigned long this_bio_flag = 0;
2911
2912         set_page_extent_mapped(page);
2913
2914         if (!PageUptodate(page)) {
2915                 if (cleancache_get_page(page) == 0) {
2916                         BUG_ON(blocksize != PAGE_SIZE);
2917                         unlock_extent(tree, start, end);
2918                         goto out;
2919                 }
2920         }
2921
2922         if (page->index == last_byte >> PAGE_SHIFT) {
2923                 char *userpage;
2924                 size_t zero_offset = last_byte & (PAGE_SIZE - 1);
2925
2926                 if (zero_offset) {
2927                         iosize = PAGE_SIZE - zero_offset;
2928                         userpage = kmap_atomic(page);
2929                         memset(userpage + zero_offset, 0, iosize);
2930                         flush_dcache_page(page);
2931                         kunmap_atomic(userpage);
2932                 }
2933         }
2934         while (cur <= end) {
2935                 bool force_bio_submit = false;
2936                 u64 offset;
2937
2938                 if (cur >= last_byte) {
2939                         char *userpage;
2940                         struct extent_state *cached = NULL;
2941
2942                         iosize = PAGE_SIZE - pg_offset;
2943                         userpage = kmap_atomic(page);
2944                         memset(userpage + pg_offset, 0, iosize);
2945                         flush_dcache_page(page);
2946                         kunmap_atomic(userpage);
2947                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2948                                             &cached, GFP_NOFS);
2949                         unlock_extent_cached(tree, cur,
2950                                              cur + iosize - 1, &cached);
2951                         break;
2952                 }
2953                 em = __get_extent_map(inode, page, pg_offset, cur,
2954                                       end - cur + 1, get_extent, em_cached);
2955                 if (IS_ERR_OR_NULL(em)) {
2956                         SetPageError(page);
2957                         unlock_extent(tree, cur, end);
2958                         break;
2959                 }
2960                 extent_offset = cur - em->start;
2961                 BUG_ON(extent_map_end(em) <= cur);
2962                 BUG_ON(end < cur);
2963
2964                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2965                         this_bio_flag |= EXTENT_BIO_COMPRESSED;
2966                         extent_set_compress_type(&this_bio_flag,
2967                                                  em->compress_type);
2968                 }
2969
2970                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2971                 cur_end = min(extent_map_end(em) - 1, end);
2972                 iosize = ALIGN(iosize, blocksize);
2973                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2974                         disk_io_size = em->block_len;
2975                         offset = em->block_start;
2976                 } else {
2977                         offset = em->block_start + extent_offset;
2978                         disk_io_size = iosize;
2979                 }
2980                 bdev = em->bdev;
2981                 block_start = em->block_start;
2982                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2983                         block_start = EXTENT_MAP_HOLE;
2984
2985                 /*
2986                  * If we have a file range that points to a compressed extent
2987                  * and it's followed by a consecutive file range that points to
2988                  * to the same compressed extent (possibly with a different
2989                  * offset and/or length, so it either points to the whole extent
2990                  * or only part of it), we must make sure we do not submit a
2991                  * single bio to populate the pages for the 2 ranges because
2992                  * this makes the compressed extent read zero out the pages
2993                  * belonging to the 2nd range. Imagine the following scenario:
2994                  *
2995                  *  File layout
2996                  *  [0 - 8K]                     [8K - 24K]
2997                  *    |                               |
2998                  *    |                               |
2999                  * points to extent X,         points to extent X,
3000                  * offset 4K, length of 8K     offset 0, length 16K
3001                  *
3002                  * [extent X, compressed length = 4K uncompressed length = 16K]
3003                  *
3004                  * If the bio to read the compressed extent covers both ranges,
3005                  * it will decompress extent X into the pages belonging to the
3006                  * first range and then it will stop, zeroing out the remaining
3007                  * pages that belong to the other range that points to extent X.
3008                  * So here we make sure we submit 2 bios, one for the first
3009                  * range and another one for the third range. Both will target
3010                  * the same physical extent from disk, but we can't currently
3011                  * make the compressed bio endio callback populate the pages
3012                  * for both ranges because each compressed bio is tightly
3013                  * coupled with a single extent map, and each range can have
3014                  * an extent map with a different offset value relative to the
3015                  * uncompressed data of our extent and different lengths. This
3016                  * is a corner case so we prioritize correctness over
3017                  * non-optimal behavior (submitting 2 bios for the same extent).
3018                  */
3019                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3020                     prev_em_start && *prev_em_start != (u64)-1 &&
3021                     *prev_em_start != em->orig_start)
3022                         force_bio_submit = true;
3023
3024                 if (prev_em_start)
3025                         *prev_em_start = em->orig_start;
3026
3027                 free_extent_map(em);
3028                 em = NULL;
3029
3030                 /* we've found a hole, just zero and go on */
3031                 if (block_start == EXTENT_MAP_HOLE) {
3032                         char *userpage;
3033                         struct extent_state *cached = NULL;
3034
3035                         userpage = kmap_atomic(page);
3036                         memset(userpage + pg_offset, 0, iosize);
3037                         flush_dcache_page(page);
3038                         kunmap_atomic(userpage);
3039
3040                         set_extent_uptodate(tree, cur, cur + iosize - 1,
3041                                             &cached, GFP_NOFS);
3042                         unlock_extent_cached(tree, cur,
3043                                              cur + iosize - 1, &cached);
3044                         cur = cur + iosize;
3045                         pg_offset += iosize;
3046                         continue;
3047                 }
3048                 /* the get_extent function already copied into the page */
3049                 if (test_range_bit(tree, cur, cur_end,
3050                                    EXTENT_UPTODATE, 1, NULL)) {
3051                         check_page_uptodate(tree, page);
3052                         unlock_extent(tree, cur, cur + iosize - 1);
3053                         cur = cur + iosize;
3054                         pg_offset += iosize;
3055                         continue;
3056                 }
3057                 /* we have an inline extent but it didn't get marked up
3058                  * to date.  Error out
3059                  */
3060                 if (block_start == EXTENT_MAP_INLINE) {
3061                         SetPageError(page);
3062                         unlock_extent(tree, cur, cur + iosize - 1);
3063                         cur = cur + iosize;
3064                         pg_offset += iosize;
3065                         continue;
3066                 }
3067
3068                 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
3069                                          page, offset, disk_io_size,
3070                                          pg_offset, bdev, bio,
3071                                          end_bio_extent_readpage, mirror_num,
3072                                          *bio_flags,
3073                                          this_bio_flag,
3074                                          force_bio_submit);
3075                 if (!ret) {
3076                         nr++;
3077                         *bio_flags = this_bio_flag;
3078                 } else {
3079                         SetPageError(page);
3080                         unlock_extent(tree, cur, cur + iosize - 1);
3081                         goto out;
3082                 }
3083                 cur = cur + iosize;
3084                 pg_offset += iosize;
3085         }
3086 out:
3087         if (!nr) {
3088                 if (!PageError(page))
3089                         SetPageUptodate(page);
3090                 unlock_page(page);
3091         }
3092         return ret;
3093 }
3094
3095 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3096                                              struct page *pages[], int nr_pages,
3097                                              u64 start, u64 end,
3098                                              struct extent_map **em_cached,
3099                                              struct bio **bio,
3100                                              unsigned long *bio_flags,
3101                                              u64 *prev_em_start)
3102 {
3103         struct inode *inode;
3104         struct btrfs_ordered_extent *ordered;
3105         int index;
3106
3107         inode = pages[0]->mapping->host;
3108         while (1) {
3109                 lock_extent(tree, start, end);
3110                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
3111                                                      end - start + 1);
3112                 if (!ordered)
3113                         break;
3114                 unlock_extent(tree, start, end);
3115                 btrfs_start_ordered_extent(inode, ordered, 1);
3116                 btrfs_put_ordered_extent(ordered);
3117         }
3118
3119         for (index = 0; index < nr_pages; index++) {
3120                 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
3121                                 bio, 0, bio_flags, 0, prev_em_start);
3122                 put_page(pages[index]);