btrfs: rename btrfs_release_extent_buffer_page
[sfrench/cifs-2.6.git] / fs / btrfs / extent_io.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/prefetch.h>
15 #include <linux/cleancache.h>
16 #include "extent_io.h"
17 #include "extent_map.h"
18 #include "ctree.h"
19 #include "btrfs_inode.h"
20 #include "volumes.h"
21 #include "check-integrity.h"
22 #include "locking.h"
23 #include "rcu-string.h"
24 #include "backref.h"
25 #include "disk-io.h"
26
27 static struct kmem_cache *extent_state_cache;
28 static struct kmem_cache *extent_buffer_cache;
29 static struct bio_set btrfs_bioset;
30
31 static inline bool extent_state_in_tree(const struct extent_state *state)
32 {
33         return !RB_EMPTY_NODE(&state->rb_node);
34 }
35
36 #ifdef CONFIG_BTRFS_DEBUG
37 static LIST_HEAD(buffers);
38 static LIST_HEAD(states);
39
40 static DEFINE_SPINLOCK(leak_lock);
41
42 static inline
43 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
44 {
45         unsigned long flags;
46
47         spin_lock_irqsave(&leak_lock, flags);
48         list_add(new, head);
49         spin_unlock_irqrestore(&leak_lock, flags);
50 }
51
52 static inline
53 void btrfs_leak_debug_del(struct list_head *entry)
54 {
55         unsigned long flags;
56
57         spin_lock_irqsave(&leak_lock, flags);
58         list_del(entry);
59         spin_unlock_irqrestore(&leak_lock, flags);
60 }
61
62 static inline
63 void btrfs_leak_debug_check(void)
64 {
65         struct extent_state *state;
66         struct extent_buffer *eb;
67
68         while (!list_empty(&states)) {
69                 state = list_entry(states.next, struct extent_state, leak_list);
70                 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
71                        state->start, state->end, state->state,
72                        extent_state_in_tree(state),
73                        refcount_read(&state->refs));
74                 list_del(&state->leak_list);
75                 kmem_cache_free(extent_state_cache, state);
76         }
77
78         while (!list_empty(&buffers)) {
79                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
80                 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81                        eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
82                 list_del(&eb->leak_list);
83                 kmem_cache_free(extent_buffer_cache, eb);
84         }
85 }
86
87 #define btrfs_debug_check_extent_io_range(tree, start, end)             \
88         __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
89 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
90                 struct extent_io_tree *tree, u64 start, u64 end)
91 {
92         if (tree->ops && tree->ops->check_extent_io_range)
93                 tree->ops->check_extent_io_range(tree->private_data, caller,
94                                                  start, end);
95 }
96 #else
97 #define btrfs_leak_debug_add(new, head) do {} while (0)
98 #define btrfs_leak_debug_del(entry)     do {} while (0)
99 #define btrfs_leak_debug_check()        do {} while (0)
100 #define btrfs_debug_check_extent_io_range(c, s, e)      do {} while (0)
101 #endif
102
103 #define BUFFER_LRU_MAX 64
104
105 struct tree_entry {
106         u64 start;
107         u64 end;
108         struct rb_node rb_node;
109 };
110
111 struct extent_page_data {
112         struct bio *bio;
113         struct extent_io_tree *tree;
114         /* tells writepage not to lock the state bits for this range
115          * it still does the unlocking
116          */
117         unsigned int extent_locked:1;
118
119         /* tells the submit_bio code to use REQ_SYNC */
120         unsigned int sync_io:1;
121 };
122
123 static int add_extent_changeset(struct extent_state *state, unsigned bits,
124                                  struct extent_changeset *changeset,
125                                  int set)
126 {
127         int ret;
128
129         if (!changeset)
130                 return 0;
131         if (set && (state->state & bits) == bits)
132                 return 0;
133         if (!set && (state->state & bits) == 0)
134                 return 0;
135         changeset->bytes_changed += state->end - state->start + 1;
136         ret = ulist_add(&changeset->range_changed, state->start, state->end,
137                         GFP_ATOMIC);
138         return ret;
139 }
140
141 static void flush_write_bio(struct extent_page_data *epd);
142
143 static inline struct btrfs_fs_info *
144 tree_fs_info(struct extent_io_tree *tree)
145 {
146         if (tree->ops)
147                 return tree->ops->tree_fs_info(tree->private_data);
148         return NULL;
149 }
150
151 int __init extent_io_init(void)
152 {
153         extent_state_cache = kmem_cache_create("btrfs_extent_state",
154                         sizeof(struct extent_state), 0,
155                         SLAB_MEM_SPREAD, NULL);
156         if (!extent_state_cache)
157                 return -ENOMEM;
158
159         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
160                         sizeof(struct extent_buffer), 0,
161                         SLAB_MEM_SPREAD, NULL);
162         if (!extent_buffer_cache)
163                 goto free_state_cache;
164
165         if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
166                         offsetof(struct btrfs_io_bio, bio),
167                         BIOSET_NEED_BVECS))
168                 goto free_buffer_cache;
169
170         if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
171                 goto free_bioset;
172
173         return 0;
174
175 free_bioset:
176         bioset_exit(&btrfs_bioset);
177
178 free_buffer_cache:
179         kmem_cache_destroy(extent_buffer_cache);
180         extent_buffer_cache = NULL;
181
182 free_state_cache:
183         kmem_cache_destroy(extent_state_cache);
184         extent_state_cache = NULL;
185         return -ENOMEM;
186 }
187
188 void __cold extent_io_exit(void)
189 {
190         btrfs_leak_debug_check();
191
192         /*
193          * Make sure all delayed rcu free are flushed before we
194          * destroy caches.
195          */
196         rcu_barrier();
197         kmem_cache_destroy(extent_state_cache);
198         kmem_cache_destroy(extent_buffer_cache);
199         bioset_exit(&btrfs_bioset);
200 }
201
202 void extent_io_tree_init(struct extent_io_tree *tree,
203                          void *private_data)
204 {
205         tree->state = RB_ROOT;
206         tree->ops = NULL;
207         tree->dirty_bytes = 0;
208         spin_lock_init(&tree->lock);
209         tree->private_data = private_data;
210 }
211
212 static struct extent_state *alloc_extent_state(gfp_t mask)
213 {
214         struct extent_state *state;
215
216         /*
217          * The given mask might be not appropriate for the slab allocator,
218          * drop the unsupported bits
219          */
220         mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
221         state = kmem_cache_alloc(extent_state_cache, mask);
222         if (!state)
223                 return state;
224         state->state = 0;
225         state->failrec = NULL;
226         RB_CLEAR_NODE(&state->rb_node);
227         btrfs_leak_debug_add(&state->leak_list, &states);
228         refcount_set(&state->refs, 1);
229         init_waitqueue_head(&state->wq);
230         trace_alloc_extent_state(state, mask, _RET_IP_);
231         return state;
232 }
233
234 void free_extent_state(struct extent_state *state)
235 {
236         if (!state)
237                 return;
238         if (refcount_dec_and_test(&state->refs)) {
239                 WARN_ON(extent_state_in_tree(state));
240                 btrfs_leak_debug_del(&state->leak_list);
241                 trace_free_extent_state(state, _RET_IP_);
242                 kmem_cache_free(extent_state_cache, state);
243         }
244 }
245
246 static struct rb_node *tree_insert(struct rb_root *root,
247                                    struct rb_node *search_start,
248                                    u64 offset,
249                                    struct rb_node *node,
250                                    struct rb_node ***p_in,
251                                    struct rb_node **parent_in)
252 {
253         struct rb_node **p;
254         struct rb_node *parent = NULL;
255         struct tree_entry *entry;
256
257         if (p_in && parent_in) {
258                 p = *p_in;
259                 parent = *parent_in;
260                 goto do_insert;
261         }
262
263         p = search_start ? &search_start : &root->rb_node;
264         while (*p) {
265                 parent = *p;
266                 entry = rb_entry(parent, struct tree_entry, rb_node);
267
268                 if (offset < entry->start)
269                         p = &(*p)->rb_left;
270                 else if (offset > entry->end)
271                         p = &(*p)->rb_right;
272                 else
273                         return parent;
274         }
275
276 do_insert:
277         rb_link_node(node, parent, p);
278         rb_insert_color(node, root);
279         return NULL;
280 }
281
282 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
283                                       struct rb_node **prev_ret,
284                                       struct rb_node **next_ret,
285                                       struct rb_node ***p_ret,
286                                       struct rb_node **parent_ret)
287 {
288         struct rb_root *root = &tree->state;
289         struct rb_node **n = &root->rb_node;
290         struct rb_node *prev = NULL;
291         struct rb_node *orig_prev = NULL;
292         struct tree_entry *entry;
293         struct tree_entry *prev_entry = NULL;
294
295         while (*n) {
296                 prev = *n;
297                 entry = rb_entry(prev, struct tree_entry, rb_node);
298                 prev_entry = entry;
299
300                 if (offset < entry->start)
301                         n = &(*n)->rb_left;
302                 else if (offset > entry->end)
303                         n = &(*n)->rb_right;
304                 else
305                         return *n;
306         }
307
308         if (p_ret)
309                 *p_ret = n;
310         if (parent_ret)
311                 *parent_ret = prev;
312
313         if (prev_ret) {
314                 orig_prev = prev;
315                 while (prev && offset > prev_entry->end) {
316                         prev = rb_next(prev);
317                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
318                 }
319                 *prev_ret = prev;
320                 prev = orig_prev;
321         }
322
323         if (next_ret) {
324                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
325                 while (prev && offset < prev_entry->start) {
326                         prev = rb_prev(prev);
327                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
328                 }
329                 *next_ret = prev;
330         }
331         return NULL;
332 }
333
334 static inline struct rb_node *
335 tree_search_for_insert(struct extent_io_tree *tree,
336                        u64 offset,
337                        struct rb_node ***p_ret,
338                        struct rb_node **parent_ret)
339 {
340         struct rb_node *prev = NULL;
341         struct rb_node *ret;
342
343         ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
344         if (!ret)
345                 return prev;
346         return ret;
347 }
348
349 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
350                                           u64 offset)
351 {
352         return tree_search_for_insert(tree, offset, NULL, NULL);
353 }
354
355 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
356                      struct extent_state *other)
357 {
358         if (tree->ops && tree->ops->merge_extent_hook)
359                 tree->ops->merge_extent_hook(tree->private_data, new, other);
360 }
361
362 /*
363  * utility function to look for merge candidates inside a given range.
364  * Any extents with matching state are merged together into a single
365  * extent in the tree.  Extents with EXTENT_IO in their state field
366  * are not merged because the end_io handlers need to be able to do
367  * operations on them without sleeping (or doing allocations/splits).
368  *
369  * This should be called with the tree lock held.
370  */
371 static void merge_state(struct extent_io_tree *tree,
372                         struct extent_state *state)
373 {
374         struct extent_state *other;
375         struct rb_node *other_node;
376
377         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
378                 return;
379
380         other_node = rb_prev(&state->rb_node);
381         if (other_node) {
382                 other = rb_entry(other_node, struct extent_state, rb_node);
383                 if (other->end == state->start - 1 &&
384                     other->state == state->state) {
385                         merge_cb(tree, state, other);
386                         state->start = other->start;
387                         rb_erase(&other->rb_node, &tree->state);
388                         RB_CLEAR_NODE(&other->rb_node);
389                         free_extent_state(other);
390                 }
391         }
392         other_node = rb_next(&state->rb_node);
393         if (other_node) {
394                 other = rb_entry(other_node, struct extent_state, rb_node);
395                 if (other->start == state->end + 1 &&
396                     other->state == state->state) {
397                         merge_cb(tree, state, other);
398                         state->end = other->end;
399                         rb_erase(&other->rb_node, &tree->state);
400                         RB_CLEAR_NODE(&other->rb_node);
401                         free_extent_state(other);
402                 }
403         }
404 }
405
406 static void set_state_cb(struct extent_io_tree *tree,
407                          struct extent_state *state, unsigned *bits)
408 {
409         if (tree->ops && tree->ops->set_bit_hook)
410                 tree->ops->set_bit_hook(tree->private_data, state, bits);
411 }
412
413 static void clear_state_cb(struct extent_io_tree *tree,
414                            struct extent_state *state, unsigned *bits)
415 {
416         if (tree->ops && tree->ops->clear_bit_hook)
417                 tree->ops->clear_bit_hook(tree->private_data, state, bits);
418 }
419
420 static void set_state_bits(struct extent_io_tree *tree,
421                            struct extent_state *state, unsigned *bits,
422                            struct extent_changeset *changeset);
423
424 /*
425  * insert an extent_state struct into the tree.  'bits' are set on the
426  * struct before it is inserted.
427  *
428  * This may return -EEXIST if the extent is already there, in which case the
429  * state struct is freed.
430  *
431  * The tree lock is not taken internally.  This is a utility function and
432  * probably isn't what you want to call (see set/clear_extent_bit).
433  */
434 static int insert_state(struct extent_io_tree *tree,
435                         struct extent_state *state, u64 start, u64 end,
436                         struct rb_node ***p,
437                         struct rb_node **parent,
438                         unsigned *bits, struct extent_changeset *changeset)
439 {
440         struct rb_node *node;
441
442         if (end < start)
443                 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
444                        end, start);
445         state->start = start;
446         state->end = end;
447
448         set_state_bits(tree, state, bits, changeset);
449
450         node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
451         if (node) {
452                 struct extent_state *found;
453                 found = rb_entry(node, struct extent_state, rb_node);
454                 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
455                        found->start, found->end, start, end);
456                 return -EEXIST;
457         }
458         merge_state(tree, state);
459         return 0;
460 }
461
462 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
463                      u64 split)
464 {
465         if (tree->ops && tree->ops->split_extent_hook)
466                 tree->ops->split_extent_hook(tree->private_data, orig, split);
467 }
468
469 /*
470  * split a given extent state struct in two, inserting the preallocated
471  * struct 'prealloc' as the newly created second half.  'split' indicates an
472  * offset inside 'orig' where it should be split.
473  *
474  * Before calling,
475  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
476  * are two extent state structs in the tree:
477  * prealloc: [orig->start, split - 1]
478  * orig: [ split, orig->end ]
479  *
480  * The tree locks are not taken by this function. They need to be held
481  * by the caller.
482  */
483 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
484                        struct extent_state *prealloc, u64 split)
485 {
486         struct rb_node *node;
487
488         split_cb(tree, orig, split);
489
490         prealloc->start = orig->start;
491         prealloc->end = split - 1;
492         prealloc->state = orig->state;
493         orig->start = split;
494
495         node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
496                            &prealloc->rb_node, NULL, NULL);
497         if (node) {
498                 free_extent_state(prealloc);
499                 return -EEXIST;
500         }
501         return 0;
502 }
503
504 static struct extent_state *next_state(struct extent_state *state)
505 {
506         struct rb_node *next = rb_next(&state->rb_node);
507         if (next)
508                 return rb_entry(next, struct extent_state, rb_node);
509         else
510                 return NULL;
511 }
512
513 /*
514  * utility function to clear some bits in an extent state struct.
515  * it will optionally wake up any one waiting on this state (wake == 1).
516  *
517  * If no bits are set on the state struct after clearing things, the
518  * struct is freed and removed from the tree
519  */
520 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
521                                             struct extent_state *state,
522                                             unsigned *bits, int wake,
523                                             struct extent_changeset *changeset)
524 {
525         struct extent_state *next;
526         unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
527         int ret;
528
529         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
530                 u64 range = state->end - state->start + 1;
531                 WARN_ON(range > tree->dirty_bytes);
532                 tree->dirty_bytes -= range;
533         }
534         clear_state_cb(tree, state, bits);
535         ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
536         BUG_ON(ret < 0);
537         state->state &= ~bits_to_clear;
538         if (wake)
539                 wake_up(&state->wq);
540         if (state->state == 0) {
541                 next = next_state(state);
542                 if (extent_state_in_tree(state)) {
543                         rb_erase(&state->rb_node, &tree->state);
544                         RB_CLEAR_NODE(&state->rb_node);
545                         free_extent_state(state);
546                 } else {
547                         WARN_ON(1);
548                 }
549         } else {
550                 merge_state(tree, state);
551                 next = next_state(state);
552         }
553         return next;
554 }
555
556 static struct extent_state *
557 alloc_extent_state_atomic(struct extent_state *prealloc)
558 {
559         if (!prealloc)
560                 prealloc = alloc_extent_state(GFP_ATOMIC);
561
562         return prealloc;
563 }
564
565 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
566 {
567         btrfs_panic(tree_fs_info(tree), err,
568                     "Locking error: Extent tree was modified by another thread while locked.");
569 }
570
571 /*
572  * clear some bits on a range in the tree.  This may require splitting
573  * or inserting elements in the tree, so the gfp mask is used to
574  * indicate which allocations or sleeping are allowed.
575  *
576  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
577  * the given range from the tree regardless of state (ie for truncate).
578  *
579  * the range [start, end] is inclusive.
580  *
581  * This takes the tree lock, and returns 0 on success and < 0 on error.
582  */
583 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
584                               unsigned bits, int wake, int delete,
585                               struct extent_state **cached_state,
586                               gfp_t mask, struct extent_changeset *changeset)
587 {
588         struct extent_state *state;
589         struct extent_state *cached;
590         struct extent_state *prealloc = NULL;
591         struct rb_node *node;
592         u64 last_end;
593         int err;
594         int clear = 0;
595
596         btrfs_debug_check_extent_io_range(tree, start, end);
597
598         if (bits & EXTENT_DELALLOC)
599                 bits |= EXTENT_NORESERVE;
600
601         if (delete)
602                 bits |= ~EXTENT_CTLBITS;
603         bits |= EXTENT_FIRST_DELALLOC;
604
605         if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
606                 clear = 1;
607 again:
608         if (!prealloc && gfpflags_allow_blocking(mask)) {
609                 /*
610                  * Don't care for allocation failure here because we might end
611                  * up not needing the pre-allocated extent state at all, which
612                  * is the case if we only have in the tree extent states that
613                  * cover our input range and don't cover too any other range.
614                  * If we end up needing a new extent state we allocate it later.
615                  */
616                 prealloc = alloc_extent_state(mask);
617         }
618
619         spin_lock(&tree->lock);
620         if (cached_state) {
621                 cached = *cached_state;
622
623                 if (clear) {
624                         *cached_state = NULL;
625                         cached_state = NULL;
626                 }
627
628                 if (cached && extent_state_in_tree(cached) &&
629                     cached->start <= start && cached->end > start) {
630                         if (clear)
631                                 refcount_dec(&cached->refs);
632                         state = cached;
633                         goto hit_next;
634                 }
635                 if (clear)
636                         free_extent_state(cached);
637         }
638         /*
639          * this search will find the extents that end after
640          * our range starts
641          */
642         node = tree_search(tree, start);
643         if (!node)
644                 goto out;
645         state = rb_entry(node, struct extent_state, rb_node);
646 hit_next:
647         if (state->start > end)
648                 goto out;
649         WARN_ON(state->end < start);
650         last_end = state->end;
651
652         /* the state doesn't have the wanted bits, go ahead */
653         if (!(state->state & bits)) {
654                 state = next_state(state);
655                 goto next;
656         }
657
658         /*
659          *     | ---- desired range ---- |
660          *  | state | or
661          *  | ------------- state -------------- |
662          *
663          * We need to split the extent we found, and may flip
664          * bits on second half.
665          *
666          * If the extent we found extends past our range, we
667          * just split and search again.  It'll get split again
668          * the next time though.
669          *
670          * If the extent we found is inside our range, we clear
671          * the desired bit on it.
672          */
673
674         if (state->start < start) {
675                 prealloc = alloc_extent_state_atomic(prealloc);
676                 BUG_ON(!prealloc);
677                 err = split_state(tree, state, prealloc, start);
678                 if (err)
679                         extent_io_tree_panic(tree, err);
680
681                 prealloc = NULL;
682                 if (err)
683                         goto out;
684                 if (state->end <= end) {
685                         state = clear_state_bit(tree, state, &bits, wake,
686                                                 changeset);
687                         goto next;
688                 }
689                 goto search_again;
690         }
691         /*
692          * | ---- desired range ---- |
693          *                        | state |
694          * We need to split the extent, and clear the bit
695          * on the first half
696          */
697         if (state->start <= end && state->end > end) {
698                 prealloc = alloc_extent_state_atomic(prealloc);
699                 BUG_ON(!prealloc);
700                 err = split_state(tree, state, prealloc, end + 1);
701                 if (err)
702                         extent_io_tree_panic(tree, err);
703
704                 if (wake)
705                         wake_up(&state->wq);
706
707                 clear_state_bit(tree, prealloc, &bits, wake, changeset);
708
709                 prealloc = NULL;
710                 goto out;
711         }
712
713         state = clear_state_bit(tree, state, &bits, wake, changeset);
714 next:
715         if (last_end == (u64)-1)
716                 goto out;
717         start = last_end + 1;
718         if (start <= end && state && !need_resched())
719                 goto hit_next;
720
721 search_again:
722         if (start > end)
723                 goto out;
724         spin_unlock(&tree->lock);
725         if (gfpflags_allow_blocking(mask))
726                 cond_resched();
727         goto again;
728
729 out:
730         spin_unlock(&tree->lock);
731         if (prealloc)
732                 free_extent_state(prealloc);
733
734         return 0;
735
736 }
737
738 static void wait_on_state(struct extent_io_tree *tree,
739                           struct extent_state *state)
740                 __releases(tree->lock)
741                 __acquires(tree->lock)
742 {
743         DEFINE_WAIT(wait);
744         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
745         spin_unlock(&tree->lock);
746         schedule();
747         spin_lock(&tree->lock);
748         finish_wait(&state->wq, &wait);
749 }
750
751 /*
752  * waits for one or more bits to clear on a range in the state tree.
753  * The range [start, end] is inclusive.
754  * The tree lock is taken by this function
755  */
756 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
757                             unsigned long bits)
758 {
759         struct extent_state *state;
760         struct rb_node *node;
761
762         btrfs_debug_check_extent_io_range(tree, start, end);
763
764         spin_lock(&tree->lock);
765 again:
766         while (1) {
767                 /*
768                  * this search will find all the extents that end after
769                  * our range starts
770                  */
771                 node = tree_search(tree, start);
772 process_node:
773                 if (!node)
774                         break;
775
776                 state = rb_entry(node, struct extent_state, rb_node);
777
778                 if (state->start > end)
779                         goto out;
780
781                 if (state->state & bits) {
782                         start = state->start;
783                         refcount_inc(&state->refs);
784                         wait_on_state(tree, state);
785                         free_extent_state(state);
786                         goto again;
787                 }
788                 start = state->end + 1;
789
790                 if (start > end)
791                         break;
792
793                 if (!cond_resched_lock(&tree->lock)) {
794                         node = rb_next(node);
795                         goto process_node;
796                 }
797         }
798 out:
799         spin_unlock(&tree->lock);
800 }
801
802 static void set_state_bits(struct extent_io_tree *tree,
803                            struct extent_state *state,
804                            unsigned *bits, struct extent_changeset *changeset)
805 {
806         unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
807         int ret;
808
809         set_state_cb(tree, state, bits);
810         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
811                 u64 range = state->end - state->start + 1;
812                 tree->dirty_bytes += range;
813         }
814         ret = add_extent_changeset(state, bits_to_set, changeset, 1);
815         BUG_ON(ret < 0);
816         state->state |= bits_to_set;
817 }
818
819 static void cache_state_if_flags(struct extent_state *state,
820                                  struct extent_state **cached_ptr,
821                                  unsigned flags)
822 {
823         if (cached_ptr && !(*cached_ptr)) {
824                 if (!flags || (state->state & flags)) {
825                         *cached_ptr = state;
826                         refcount_inc(&state->refs);
827                 }
828         }
829 }
830
831 static void cache_state(struct extent_state *state,
832                         struct extent_state **cached_ptr)
833 {
834         return cache_state_if_flags(state, cached_ptr,
835                                     EXTENT_IOBITS | EXTENT_BOUNDARY);
836 }
837
838 /*
839  * set some bits on a range in the tree.  This may require allocations or
840  * sleeping, so the gfp mask is used to indicate what is allowed.
841  *
842  * If any of the exclusive bits are set, this will fail with -EEXIST if some
843  * part of the range already has the desired bits set.  The start of the
844  * existing range is returned in failed_start in this case.
845  *
846  * [start, end] is inclusive This takes the tree lock.
847  */
848
849 static int __must_check
850 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
851                  unsigned bits, unsigned exclusive_bits,
852                  u64 *failed_start, struct extent_state **cached_state,
853                  gfp_t mask, struct extent_changeset *changeset)
854 {
855         struct extent_state *state;
856         struct extent_state *prealloc = NULL;
857         struct rb_node *node;
858         struct rb_node **p;
859         struct rb_node *parent;
860         int err = 0;
861         u64 last_start;
862         u64 last_end;
863
864         btrfs_debug_check_extent_io_range(tree, start, end);
865
866         bits |= EXTENT_FIRST_DELALLOC;
867 again:
868         if (!prealloc && gfpflags_allow_blocking(mask)) {
869                 /*
870                  * Don't care for allocation failure here because we might end
871                  * up not needing the pre-allocated extent state at all, which
872                  * is the case if we only have in the tree extent states that
873                  * cover our input range and don't cover too any other range.
874                  * If we end up needing a new extent state we allocate it later.
875                  */
876                 prealloc = alloc_extent_state(mask);
877         }
878
879         spin_lock(&tree->lock);
880         if (cached_state && *cached_state) {
881                 state = *cached_state;
882                 if (state->start <= start && state->end > start &&
883                     extent_state_in_tree(state)) {
884                         node = &state->rb_node;
885                         goto hit_next;
886                 }
887         }
888         /*
889          * this search will find all the extents that end after
890          * our range starts.
891          */
892         node = tree_search_for_insert(tree, start, &p, &parent);
893         if (!node) {
894                 prealloc = alloc_extent_state_atomic(prealloc);
895                 BUG_ON(!prealloc);
896                 err = insert_state(tree, prealloc, start, end,
897                                    &p, &parent, &bits, changeset);
898                 if (err)
899                         extent_io_tree_panic(tree, err);
900
901                 cache_state(prealloc, cached_state);
902                 prealloc = NULL;
903                 goto out;
904         }
905         state = rb_entry(node, struct extent_state, rb_node);
906 hit_next:
907         last_start = state->start;
908         last_end = state->end;
909
910         /*
911          * | ---- desired range ---- |
912          * | state |
913          *
914          * Just lock what we found and keep going
915          */
916         if (state->start == start && state->end <= end) {
917                 if (state->state & exclusive_bits) {
918                         *failed_start = state->start;
919                         err = -EEXIST;
920                         goto out;
921                 }
922
923                 set_state_bits(tree, state, &bits, changeset);
924                 cache_state(state, cached_state);
925                 merge_state(tree, state);
926                 if (last_end == (u64)-1)
927                         goto out;
928                 start = last_end + 1;
929                 state = next_state(state);
930                 if (start < end && state && state->start == start &&
931                     !need_resched())
932                         goto hit_next;
933                 goto search_again;
934         }
935
936         /*
937          *     | ---- desired range ---- |
938          * | state |
939          *   or
940          * | ------------- state -------------- |
941          *
942          * We need to split the extent we found, and may flip bits on
943          * second half.
944          *
945          * If the extent we found extends past our
946          * range, we just split and search again.  It'll get split
947          * again the next time though.
948          *
949          * If the extent we found is inside our range, we set the
950          * desired bit on it.
951          */
952         if (state->start < start) {
953                 if (state->state & exclusive_bits) {
954                         *failed_start = start;
955                         err = -EEXIST;
956                         goto out;
957                 }
958
959                 prealloc = alloc_extent_state_atomic(prealloc);
960                 BUG_ON(!prealloc);
961                 err = split_state(tree, state, prealloc, start);
962                 if (err)
963                         extent_io_tree_panic(tree, err);
964
965                 prealloc = NULL;
966                 if (err)
967                         goto out;
968                 if (state->end <= end) {
969                         set_state_bits(tree, state, &bits, changeset);
970                         cache_state(state, cached_state);
971                         merge_state(tree, state);
972                         if (last_end == (u64)-1)
973                                 goto out;
974                         start = last_end + 1;
975                         state = next_state(state);
976                         if (start < end && state && state->start == start &&
977                             !need_resched())
978                                 goto hit_next;
979                 }
980                 goto search_again;
981         }
982         /*
983          * | ---- desired range ---- |
984          *     | state | or               | state |
985          *
986          * There's a hole, we need to insert something in it and
987          * ignore the extent we found.
988          */
989         if (state->start > start) {
990                 u64 this_end;
991                 if (end < last_start)
992                         this_end = end;
993                 else
994                         this_end = last_start - 1;
995
996                 prealloc = alloc_extent_state_atomic(prealloc);
997                 BUG_ON(!prealloc);
998
999                 /*
1000                  * Avoid to free 'prealloc' if it can be merged with
1001                  * the later extent.
1002                  */
1003                 err = insert_state(tree, prealloc, start, this_end,
1004                                    NULL, NULL, &bits, changeset);
1005                 if (err)
1006                         extent_io_tree_panic(tree, err);
1007
1008                 cache_state(prealloc, cached_state);
1009                 prealloc = NULL;
1010                 start = this_end + 1;
1011                 goto search_again;
1012         }
1013         /*
1014          * | ---- desired range ---- |
1015          *                        | state |
1016          * We need to split the extent, and set the bit
1017          * on the first half
1018          */
1019         if (state->start <= end && state->end > end) {
1020                 if (state->state & exclusive_bits) {
1021                         *failed_start = start;
1022                         err = -EEXIST;
1023                         goto out;
1024                 }
1025
1026                 prealloc = alloc_extent_state_atomic(prealloc);
1027                 BUG_ON(!prealloc);
1028                 err = split_state(tree, state, prealloc, end + 1);
1029                 if (err)
1030                         extent_io_tree_panic(tree, err);
1031
1032                 set_state_bits(tree, prealloc, &bits, changeset);
1033                 cache_state(prealloc, cached_state);
1034                 merge_state(tree, prealloc);
1035                 prealloc = NULL;
1036                 goto out;
1037         }
1038
1039 search_again:
1040         if (start > end)
1041                 goto out;
1042         spin_unlock(&tree->lock);
1043         if (gfpflags_allow_blocking(mask))
1044                 cond_resched();
1045         goto again;
1046
1047 out:
1048         spin_unlock(&tree->lock);
1049         if (prealloc)
1050                 free_extent_state(prealloc);
1051
1052         return err;
1053
1054 }
1055
1056 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1057                    unsigned bits, u64 * failed_start,
1058                    struct extent_state **cached_state, gfp_t mask)
1059 {
1060         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1061                                 cached_state, mask, NULL);
1062 }
1063
1064
1065 /**
1066  * convert_extent_bit - convert all bits in a given range from one bit to
1067  *                      another
1068  * @tree:       the io tree to search
1069  * @start:      the start offset in bytes
1070  * @end:        the end offset in bytes (inclusive)
1071  * @bits:       the bits to set in this range
1072  * @clear_bits: the bits to clear in this range
1073  * @cached_state:       state that we're going to cache
1074  *
1075  * This will go through and set bits for the given range.  If any states exist
1076  * already in this range they are set with the given bit and cleared of the
1077  * clear_bits.  This is only meant to be used by things that are mergeable, ie
1078  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1079  * boundary bits like LOCK.
1080  *
1081  * All allocations are done with GFP_NOFS.
1082  */
1083 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1084                        unsigned bits, unsigned clear_bits,
1085                        struct extent_state **cached_state)
1086 {
1087         struct extent_state *state;
1088         struct extent_state *prealloc = NULL;
1089         struct rb_node *node;
1090         struct rb_node **p;
1091         struct rb_node *parent;
1092         int err = 0;
1093         u64 last_start;
1094         u64 last_end;
1095         bool first_iteration = true;
1096
1097         btrfs_debug_check_extent_io_range(tree, start, end);
1098
1099 again:
1100         if (!prealloc) {
1101                 /*
1102                  * Best effort, don't worry if extent state allocation fails
1103                  * here for the first iteration. We might have a cached state
1104                  * that matches exactly the target range, in which case no
1105                  * extent state allocations are needed. We'll only know this
1106                  * after locking the tree.
1107                  */
1108                 prealloc = alloc_extent_state(GFP_NOFS);
1109                 if (!prealloc && !first_iteration)
1110                         return -ENOMEM;
1111         }
1112
1113         spin_lock(&tree->lock);
1114         if (cached_state && *cached_state) {
1115                 state = *cached_state;
1116                 if (state->start <= start && state->end > start &&
1117                     extent_state_in_tree(state)) {
1118                         node = &state->rb_node;
1119                         goto hit_next;
1120                 }
1121         }
1122
1123         /*
1124          * this search will find all the extents that end after
1125          * our range starts.
1126          */
1127         node = tree_search_for_insert(tree, start, &p, &parent);
1128         if (!node) {
1129                 prealloc = alloc_extent_state_atomic(prealloc);
1130                 if (!prealloc) {
1131                         err = -ENOMEM;
1132                         goto out;
1133                 }
1134                 err = insert_state(tree, prealloc, start, end,
1135                                    &p, &parent, &bits, NULL);
1136                 if (err)
1137                         extent_io_tree_panic(tree, err);
1138                 cache_state(prealloc, cached_state);
1139                 prealloc = NULL;
1140                 goto out;
1141         }
1142         state = rb_entry(node, struct extent_state, rb_node);
1143 hit_next:
1144         last_start = state->start;
1145         last_end = state->end;
1146
1147         /*
1148          * | ---- desired range ---- |
1149          * | state |
1150          *
1151          * Just lock what we found and keep going
1152          */
1153         if (state->start == start && state->end <= end) {
1154                 set_state_bits(tree, state, &bits, NULL);
1155                 cache_state(state, cached_state);
1156                 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1157                 if (last_end == (u64)-1)
1158                         goto out;
1159                 start = last_end + 1;
1160                 if (start < end && state && state->start == start &&
1161                     !need_resched())
1162                         goto hit_next;
1163                 goto search_again;
1164         }
1165
1166         /*
1167          *     | ---- desired range ---- |
1168          * | state |
1169          *   or
1170          * | ------------- state -------------- |
1171          *
1172          * We need to split the extent we found, and may flip bits on
1173          * second half.
1174          *
1175          * If the extent we found extends past our
1176          * range, we just split and search again.  It'll get split
1177          * again the next time though.
1178          *
1179          * If the extent we found is inside our range, we set the
1180          * desired bit on it.
1181          */
1182         if (state->start < start) {
1183                 prealloc = alloc_extent_state_atomic(prealloc);
1184                 if (!prealloc) {
1185                         err = -ENOMEM;
1186                         goto out;
1187                 }
1188                 err = split_state(tree, state, prealloc, start);
1189                 if (err)
1190                         extent_io_tree_panic(tree, err);
1191                 prealloc = NULL;
1192                 if (err)
1193                         goto out;
1194                 if (state->end <= end) {
1195                         set_state_bits(tree, state, &bits, NULL);
1196                         cache_state(state, cached_state);
1197                         state = clear_state_bit(tree, state, &clear_bits, 0,
1198                                                 NULL);
1199                         if (last_end == (u64)-1)
1200                                 goto out;
1201                         start = last_end + 1;
1202                         if (start < end && state && state->start == start &&
1203                             !need_resched())
1204                                 goto hit_next;
1205                 }
1206                 goto search_again;
1207         }
1208         /*
1209          * | ---- desired range ---- |
1210          *     | state | or               | state |
1211          *
1212          * There's a hole, we need to insert something in it and
1213          * ignore the extent we found.
1214          */
1215         if (state->start > start) {
1216                 u64 this_end;
1217                 if (end < last_start)
1218                         this_end = end;
1219                 else
1220                         this_end = last_start - 1;
1221
1222                 prealloc = alloc_extent_state_atomic(prealloc);
1223                 if (!prealloc) {
1224                         err = -ENOMEM;
1225                         goto out;
1226                 }
1227
1228                 /*
1229                  * Avoid to free 'prealloc' if it can be merged with
1230                  * the later extent.
1231                  */
1232                 err = insert_state(tree, prealloc, start, this_end,
1233                                    NULL, NULL, &bits, NULL);
1234                 if (err)
1235                         extent_io_tree_panic(tree, err);
1236                 cache_state(prealloc, cached_state);
1237                 prealloc = NULL;
1238                 start = this_end + 1;
1239                 goto search_again;
1240         }
1241         /*
1242          * | ---- desired range ---- |
1243          *                        | state |
1244          * We need to split the extent, and set the bit
1245          * on the first half
1246          */
1247         if (state->start <= end && state->end > end) {
1248                 prealloc = alloc_extent_state_atomic(prealloc);
1249                 if (!prealloc) {
1250                         err = -ENOMEM;
1251                         goto out;
1252                 }
1253
1254                 err = split_state(tree, state, prealloc, end + 1);
1255                 if (err)
1256                         extent_io_tree_panic(tree, err);
1257
1258                 set_state_bits(tree, prealloc, &bits, NULL);
1259                 cache_state(prealloc, cached_state);
1260                 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1261                 prealloc = NULL;
1262                 goto out;
1263         }
1264
1265 search_again:
1266         if (start > end)
1267                 goto out;
1268         spin_unlock(&tree->lock);
1269         cond_resched();
1270         first_iteration = false;
1271         goto again;
1272
1273 out:
1274         spin_unlock(&tree->lock);
1275         if (prealloc)
1276                 free_extent_state(prealloc);
1277
1278         return err;
1279 }
1280
1281 /* wrappers around set/clear extent bit */
1282 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1283                            unsigned bits, struct extent_changeset *changeset)
1284 {
1285         /*
1286          * We don't support EXTENT_LOCKED yet, as current changeset will
1287          * record any bits changed, so for EXTENT_LOCKED case, it will
1288          * either fail with -EEXIST or changeset will record the whole
1289          * range.
1290          */
1291         BUG_ON(bits & EXTENT_LOCKED);
1292
1293         return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1294                                 changeset);
1295 }
1296
1297 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1298                      unsigned bits, int wake, int delete,
1299                      struct extent_state **cached)
1300 {
1301         return __clear_extent_bit(tree, start, end, bits, wake, delete,
1302                                   cached, GFP_NOFS, NULL);
1303 }
1304
1305 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1306                 unsigned bits, struct extent_changeset *changeset)
1307 {
1308         /*
1309          * Don't support EXTENT_LOCKED case, same reason as
1310          * set_record_extent_bits().
1311          */
1312         BUG_ON(bits & EXTENT_LOCKED);
1313
1314         return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1315                                   changeset);
1316 }
1317
1318 /*
1319  * either insert or lock state struct between start and end use mask to tell
1320  * us if waiting is desired.
1321  */
1322 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1323                      struct extent_state **cached_state)
1324 {
1325         int err;
1326         u64 failed_start;
1327
1328         while (1) {
1329                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1330                                        EXTENT_LOCKED, &failed_start,
1331                                        cached_state, GFP_NOFS, NULL);
1332                 if (err == -EEXIST) {
1333                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1334                         start = failed_start;
1335                 } else
1336                         break;
1337                 WARN_ON(start > end);
1338         }
1339         return err;
1340 }
1341
1342 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1343 {
1344         int err;
1345         u64 failed_start;
1346
1347         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1348                                &failed_start, NULL, GFP_NOFS, NULL);
1349         if (err == -EEXIST) {
1350                 if (failed_start > start)
1351                         clear_extent_bit(tree, start, failed_start - 1,
1352                                          EXTENT_LOCKED, 1, 0, NULL);
1353                 return 0;
1354         }
1355         return 1;
1356 }
1357
1358 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1359 {
1360         unsigned long index = start >> PAGE_SHIFT;
1361         unsigned long end_index = end >> PAGE_SHIFT;
1362         struct page *page;
1363
1364         while (index <= end_index) {
1365                 page = find_get_page(inode->i_mapping, index);
1366                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1367                 clear_page_dirty_for_io(page);
1368                 put_page(page);
1369                 index++;
1370         }
1371 }
1372
1373 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1374 {
1375         unsigned long index = start >> PAGE_SHIFT;
1376         unsigned long end_index = end >> PAGE_SHIFT;
1377         struct page *page;
1378
1379         while (index <= end_index) {
1380                 page = find_get_page(inode->i_mapping, index);
1381                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1382                 __set_page_dirty_nobuffers(page);
1383                 account_page_redirty(page);
1384                 put_page(page);
1385                 index++;
1386         }
1387 }
1388
1389 /*
1390  * helper function to set both pages and extents in the tree writeback
1391  */
1392 static void set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1393 {
1394         tree->ops->set_range_writeback(tree->private_data, start, end);
1395 }
1396
1397 /* find the first state struct with 'bits' set after 'start', and
1398  * return it.  tree->lock must be held.  NULL will returned if
1399  * nothing was found after 'start'
1400  */
1401 static struct extent_state *
1402 find_first_extent_bit_state(struct extent_io_tree *tree,
1403                             u64 start, unsigned bits)
1404 {
1405         struct rb_node *node;
1406         struct extent_state *state;
1407
1408         /*
1409          * this search will find all the extents that end after
1410          * our range starts.
1411          */
1412         node = tree_search(tree, start);
1413         if (!node)
1414                 goto out;
1415
1416         while (1) {
1417                 state = rb_entry(node, struct extent_state, rb_node);
1418                 if (state->end >= start && (state->state & bits))
1419                         return state;
1420
1421                 node = rb_next(node);
1422                 if (!node)
1423                         break;
1424         }
1425 out:
1426         return NULL;
1427 }
1428
1429 /*
1430  * find the first offset in the io tree with 'bits' set. zero is
1431  * returned if we find something, and *start_ret and *end_ret are
1432  * set to reflect the state struct that was found.
1433  *
1434  * If nothing was found, 1 is returned. If found something, return 0.
1435  */
1436 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1437                           u64 *start_ret, u64 *end_ret, unsigned bits,
1438                           struct extent_state **cached_state)
1439 {
1440         struct extent_state *state;
1441         struct rb_node *n;
1442         int ret = 1;
1443
1444         spin_lock(&tree->lock);
1445         if (cached_state && *cached_state) {
1446                 state = *cached_state;
1447                 if (state->end == start - 1 && extent_state_in_tree(state)) {
1448                         n = rb_next(&state->rb_node);
1449                         while (n) {
1450                                 state = rb_entry(n, struct extent_state,
1451                                                  rb_node);
1452                                 if (state->state & bits)
1453                                         goto got_it;
1454                                 n = rb_next(n);
1455                         }
1456                         free_extent_state(*cached_state);
1457                         *cached_state = NULL;
1458                         goto out;
1459                 }
1460                 free_extent_state(*cached_state);
1461                 *cached_state = NULL;
1462         }
1463
1464         state = find_first_extent_bit_state(tree, start, bits);
1465 got_it:
1466         if (state) {
1467                 cache_state_if_flags(state, cached_state, 0);
1468                 *start_ret = state->start;
1469                 *end_ret = state->end;
1470                 ret = 0;
1471         }
1472 out:
1473         spin_unlock(&tree->lock);
1474         return ret;
1475 }
1476
1477 /*
1478  * find a contiguous range of bytes in the file marked as delalloc, not
1479  * more than 'max_bytes'.  start and end are used to return the range,
1480  *
1481  * 1 is returned if we find something, 0 if nothing was in the tree
1482  */
1483 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1484                                         u64 *start, u64 *end, u64 max_bytes,
1485                                         struct extent_state **cached_state)
1486 {
1487         struct rb_node *node;
1488         struct extent_state *state;
1489         u64 cur_start = *start;
1490         u64 found = 0;
1491         u64 total_bytes = 0;
1492
1493         spin_lock(&tree->lock);
1494
1495         /*
1496          * this search will find all the extents that end after
1497          * our range starts.
1498          */
1499         node = tree_search(tree, cur_start);
1500         if (!node) {
1501                 if (!found)
1502                         *end = (u64)-1;
1503                 goto out;
1504         }
1505
1506         while (1) {
1507                 state = rb_entry(node, struct extent_state, rb_node);
1508                 if (found && (state->start != cur_start ||
1509                               (state->state & EXTENT_BOUNDARY))) {
1510                         goto out;
1511                 }
1512                 if (!(state->state & EXTENT_DELALLOC)) {
1513                         if (!found)
1514                                 *end = state->end;
1515                         goto out;
1516                 }
1517                 if (!found) {
1518                         *start = state->start;
1519                         *cached_state = state;
1520                         refcount_inc(&state->refs);
1521                 }
1522                 found++;
1523                 *end = state->end;
1524                 cur_start = state->end + 1;
1525                 node = rb_next(node);
1526                 total_bytes += state->end - state->start + 1;
1527                 if (total_bytes >= max_bytes)
1528                         break;
1529                 if (!node)
1530                         break;
1531         }
1532 out:
1533         spin_unlock(&tree->lock);
1534         return found;
1535 }
1536
1537 static int __process_pages_contig(struct address_space *mapping,
1538                                   struct page *locked_page,
1539                                   pgoff_t start_index, pgoff_t end_index,
1540                                   unsigned long page_ops, pgoff_t *index_ret);
1541
1542 static noinline void __unlock_for_delalloc(struct inode *inode,
1543                                            struct page *locked_page,
1544                                            u64 start, u64 end)
1545 {
1546         unsigned long index = start >> PAGE_SHIFT;
1547         unsigned long end_index = end >> PAGE_SHIFT;
1548
1549         ASSERT(locked_page);
1550         if (index == locked_page->index && end_index == index)
1551                 return;
1552
1553         __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1554                                PAGE_UNLOCK, NULL);
1555 }
1556
1557 static noinline int lock_delalloc_pages(struct inode *inode,
1558                                         struct page *locked_page,
1559                                         u64 delalloc_start,
1560                                         u64 delalloc_end)
1561 {
1562         unsigned long index = delalloc_start >> PAGE_SHIFT;
1563         unsigned long index_ret = index;
1564         unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1565         int ret;
1566
1567         ASSERT(locked_page);
1568         if (index == locked_page->index && index == end_index)
1569                 return 0;
1570
1571         ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1572                                      end_index, PAGE_LOCK, &index_ret);
1573         if (ret == -EAGAIN)
1574                 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1575                                       (u64)index_ret << PAGE_SHIFT);
1576         return ret;
1577 }
1578
1579 /*
1580  * find a contiguous range of bytes in the file marked as delalloc, not
1581  * more than 'max_bytes'.  start and end are used to return the range,
1582  *
1583  * 1 is returned if we find something, 0 if nothing was in the tree
1584  */
1585 STATIC u64 find_lock_delalloc_range(struct inode *inode,
1586                                     struct extent_io_tree *tree,
1587                                     struct page *locked_page, u64 *start,
1588                                     u64 *end, u64 max_bytes)
1589 {
1590         u64 delalloc_start;
1591         u64 delalloc_end;
1592         u64 found;
1593         struct extent_state *cached_state = NULL;
1594         int ret;
1595         int loops = 0;
1596
1597 again:
1598         /* step one, find a bunch of delalloc bytes starting at start */
1599         delalloc_start = *start;
1600         delalloc_end = 0;
1601         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1602                                     max_bytes, &cached_state);
1603         if (!found || delalloc_end <= *start) {
1604                 *start = delalloc_start;
1605                 *end = delalloc_end;
1606                 free_extent_state(cached_state);
1607                 return 0;
1608         }
1609
1610         /*
1611          * start comes from the offset of locked_page.  We have to lock
1612          * pages in order, so we can't process delalloc bytes before
1613          * locked_page
1614          */
1615         if (delalloc_start < *start)
1616                 delalloc_start = *start;
1617
1618         /*
1619          * make sure to limit the number of pages we try to lock down
1620          */
1621         if (delalloc_end + 1 - delalloc_start > max_bytes)
1622                 delalloc_end = delalloc_start + max_bytes - 1;
1623
1624         /* step two, lock all the pages after the page that has start */
1625         ret = lock_delalloc_pages(inode, locked_page,
1626                                   delalloc_start, delalloc_end);
1627         if (ret == -EAGAIN) {
1628                 /* some of the pages are gone, lets avoid looping by
1629                  * shortening the size of the delalloc range we're searching
1630                  */
1631                 free_extent_state(cached_state);
1632                 cached_state = NULL;
1633                 if (!loops) {
1634                         max_bytes = PAGE_SIZE;
1635                         loops = 1;
1636                         goto again;
1637                 } else {
1638                         found = 0;
1639                         goto out_failed;
1640                 }
1641         }
1642         BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1643
1644         /* step three, lock the state bits for the whole range */
1645         lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1646
1647         /* then test to make sure it is all still delalloc */
1648         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1649                              EXTENT_DELALLOC, 1, cached_state);
1650         if (!ret) {
1651                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1652                                      &cached_state);
1653                 __unlock_for_delalloc(inode, locked_page,
1654                               delalloc_start, delalloc_end);
1655                 cond_resched();
1656                 goto again;
1657         }
1658         free_extent_state(cached_state);
1659         *start = delalloc_start;
1660         *end = delalloc_end;
1661 out_failed:
1662         return found;
1663 }
1664
1665 static int __process_pages_contig(struct address_space *mapping,
1666                                   struct page *locked_page,
1667                                   pgoff_t start_index, pgoff_t end_index,
1668                                   unsigned long page_ops, pgoff_t *index_ret)
1669 {
1670         unsigned long nr_pages = end_index - start_index + 1;
1671         unsigned long pages_locked = 0;
1672         pgoff_t index = start_index;
1673         struct page *pages[16];
1674         unsigned ret;
1675         int err = 0;
1676         int i;
1677
1678         if (page_ops & PAGE_LOCK) {
1679                 ASSERT(page_ops == PAGE_LOCK);
1680                 ASSERT(index_ret && *index_ret == start_index);
1681         }
1682
1683         if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1684                 mapping_set_error(mapping, -EIO);
1685
1686         while (nr_pages > 0) {
1687                 ret = find_get_pages_contig(mapping, index,
1688                                      min_t(unsigned long,
1689                                      nr_pages, ARRAY_SIZE(pages)), pages);
1690                 if (ret == 0) {
1691                         /*
1692                          * Only if we're going to lock these pages,
1693                          * can we find nothing at @index.
1694                          */
1695                         ASSERT(page_ops & PAGE_LOCK);
1696                         err = -EAGAIN;
1697                         goto out;
1698                 }
1699
1700                 for (i = 0; i < ret; i++) {
1701                         if (page_ops & PAGE_SET_PRIVATE2)
1702                                 SetPagePrivate2(pages[i]);
1703
1704                         if (pages[i] == locked_page) {
1705                                 put_page(pages[i]);
1706                                 pages_locked++;
1707                                 continue;
1708                         }
1709                         if (page_ops & PAGE_CLEAR_DIRTY)
1710                                 clear_page_dirty_for_io(pages[i]);
1711                         if (page_ops & PAGE_SET_WRITEBACK)
1712                                 set_page_writeback(pages[i]);
1713                         if (page_ops & PAGE_SET_ERROR)
1714                                 SetPageError(pages[i]);
1715                         if (page_ops & PAGE_END_WRITEBACK)
1716                                 end_page_writeback(pages[i]);
1717                         if (page_ops & PAGE_UNLOCK)
1718                                 unlock_page(pages[i]);
1719                         if (page_ops & PAGE_LOCK) {
1720                                 lock_page(pages[i]);
1721                                 if (!PageDirty(pages[i]) ||
1722                                     pages[i]->mapping != mapping) {
1723                                         unlock_page(pages[i]);
1724                                         put_page(pages[i]);
1725                                         err = -EAGAIN;
1726                                         goto out;
1727                                 }
1728                         }
1729                         put_page(pages[i]);
1730                         pages_locked++;
1731                 }
1732                 nr_pages -= ret;
1733                 index += ret;
1734                 cond_resched();
1735         }
1736 out:
1737         if (err && index_ret)
1738                 *index_ret = start_index + pages_locked - 1;
1739         return err;
1740 }
1741
1742 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1743                                  u64 delalloc_end, struct page *locked_page,
1744                                  unsigned clear_bits,
1745                                  unsigned long page_ops)
1746 {
1747         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
1748                          NULL);
1749
1750         __process_pages_contig(inode->i_mapping, locked_page,
1751                                start >> PAGE_SHIFT, end >> PAGE_SHIFT,
1752                                page_ops, NULL);
1753 }
1754
1755 /*
1756  * count the number of bytes in the tree that have a given bit(s)
1757  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1758  * cached.  The total number found is returned.
1759  */
1760 u64 count_range_bits(struct extent_io_tree *tree,
1761                      u64 *start, u64 search_end, u64 max_bytes,
1762                      unsigned bits, int contig)
1763 {
1764         struct rb_node *node;
1765         struct extent_state *state;
1766         u64 cur_start = *start;
1767         u64 total_bytes = 0;
1768         u64 last = 0;
1769         int found = 0;
1770
1771         if (WARN_ON(search_end <= cur_start))
1772                 return 0;
1773
1774         spin_lock(&tree->lock);
1775         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1776                 total_bytes = tree->dirty_bytes;
1777                 goto out;
1778         }
1779         /*
1780          * this search will find all the extents that end after
1781          * our range starts.
1782          */
1783         node = tree_search(tree, cur_start);
1784         if (!node)
1785                 goto out;
1786
1787         while (1) {
1788                 state = rb_entry(node, struct extent_state, rb_node);
1789                 if (state->start > search_end)
1790                         break;
1791                 if (contig && found && state->start > last + 1)
1792                         break;
1793                 if (state->end >= cur_start && (state->state & bits) == bits) {
1794                         total_bytes += min(search_end, state->end) + 1 -
1795                                        max(cur_start, state->start);
1796                         if (total_bytes >= max_bytes)
1797                                 break;
1798                         if (!found) {
1799                                 *start = max(cur_start, state->start);
1800                                 found = 1;
1801                         }
1802                         last = state->end;
1803                 } else if (contig && found) {
1804                         break;
1805                 }
1806                 node = rb_next(node);
1807                 if (!node)
1808                         break;
1809         }
1810 out:
1811         spin_unlock(&tree->lock);
1812         return total_bytes;
1813 }
1814
1815 /*
1816  * set the private field for a given byte offset in the tree.  If there isn't
1817  * an extent_state there already, this does nothing.
1818  */
1819 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
1820                 struct io_failure_record *failrec)
1821 {
1822         struct rb_node *node;
1823         struct extent_state *state;
1824         int ret = 0;
1825
1826         spin_lock(&tree->lock);
1827         /*
1828          * this search will find all the extents that end after
1829          * our range starts.
1830          */
1831         node = tree_search(tree, start);
1832         if (!node) {
1833                 ret = -ENOENT;
1834                 goto out;
1835         }
1836         state = rb_entry(node, struct extent_state, rb_node);
1837         if (state->start != start) {
1838                 ret = -ENOENT;
1839                 goto out;
1840         }
1841         state->failrec = failrec;
1842 out:
1843         spin_unlock(&tree->lock);
1844         return ret;
1845 }
1846
1847 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
1848                 struct io_failure_record **failrec)
1849 {
1850         struct rb_node *node;
1851         struct extent_state *state;
1852         int ret = 0;
1853
1854         spin_lock(&tree->lock);
1855         /*
1856          * this search will find all the extents that end after
1857          * our range starts.
1858          */
1859         node = tree_search(tree, start);
1860         if (!node) {
1861                 ret = -ENOENT;
1862                 goto out;
1863         }
1864         state = rb_entry(node, struct extent_state, rb_node);
1865         if (state->start != start) {
1866                 ret = -ENOENT;
1867                 goto out;
1868         }
1869         *failrec = state->failrec;
1870 out:
1871         spin_unlock(&tree->lock);
1872         return ret;
1873 }
1874
1875 /*
1876  * searches a range in the state tree for a given mask.
1877  * If 'filled' == 1, this returns 1 only if every extent in the tree
1878  * has the bits set.  Otherwise, 1 is returned if any bit in the
1879  * range is found set.
1880  */
1881 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1882                    unsigned bits, int filled, struct extent_state *cached)
1883 {
1884         struct extent_state *state = NULL;
1885         struct rb_node *node;
1886         int bitset = 0;
1887
1888         spin_lock(&tree->lock);
1889         if (cached && extent_state_in_tree(cached) && cached->start <= start &&
1890             cached->end > start)
1891                 node = &cached->rb_node;
1892         else
1893                 node = tree_search(tree, start);
1894         while (node && start <= end) {
1895                 state = rb_entry(node, struct extent_state, rb_node);
1896
1897                 if (filled && state->start > start) {
1898                         bitset = 0;
1899                         break;
1900                 }
1901
1902                 if (state->start > end)
1903                         break;
1904
1905                 if (state->state & bits) {
1906                         bitset = 1;
1907                         if (!filled)
1908                                 break;
1909                 } else if (filled) {
1910                         bitset = 0;
1911                         break;
1912                 }
1913
1914                 if (state->end == (u64)-1)
1915                         break;
1916
1917                 start = state->end + 1;
1918                 if (start > end)
1919                         break;
1920                 node = rb_next(node);
1921                 if (!node) {
1922                         if (filled)
1923                                 bitset = 0;
1924                         break;
1925                 }
1926         }
1927         spin_unlock(&tree->lock);
1928         return bitset;
1929 }
1930
1931 /*
1932  * helper function to set a given page up to date if all the
1933  * extents in the tree for that page are up to date
1934  */
1935 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1936 {
1937         u64 start = page_offset(page);
1938         u64 end = start + PAGE_SIZE - 1;
1939         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1940                 SetPageUptodate(page);
1941 }
1942
1943 int free_io_failure(struct extent_io_tree *failure_tree,
1944                     struct extent_io_tree *io_tree,
1945                     struct io_failure_record *rec)
1946 {
1947         int ret;
1948         int err = 0;
1949
1950         set_state_failrec(failure_tree, rec->start, NULL);
1951         ret = clear_extent_bits(failure_tree, rec->start,
1952                                 rec->start + rec->len - 1,
1953                                 EXTENT_LOCKED | EXTENT_DIRTY);
1954         if (ret)
1955                 err = ret;
1956
1957         ret = clear_extent_bits(io_tree, rec->start,
1958                                 rec->start + rec->len - 1,
1959                                 EXTENT_DAMAGED);
1960         if (ret && !err)
1961                 err = ret;
1962
1963         kfree(rec);
1964         return err;
1965 }
1966
1967 /*
1968  * this bypasses the standard btrfs submit functions deliberately, as
1969  * the standard behavior is to write all copies in a raid setup. here we only
1970  * want to write the one bad copy. so we do the mapping for ourselves and issue
1971  * submit_bio directly.
1972  * to avoid any synchronization issues, wait for the data after writing, which
1973  * actually prevents the read that triggered the error from finishing.
1974  * currently, there can be no more than two copies of every data bit. thus,
1975  * exactly one rewrite is required.
1976  */
1977 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1978                       u64 length, u64 logical, struct page *page,
1979                       unsigned int pg_offset, int mirror_num)
1980 {
1981         struct bio *bio;
1982         struct btrfs_device *dev;
1983         u64 map_length = 0;
1984         u64 sector;
1985         struct btrfs_bio *bbio = NULL;
1986         int ret;
1987
1988         ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
1989         BUG_ON(!mirror_num);
1990
1991         bio = btrfs_io_bio_alloc(1);
1992         bio->bi_iter.bi_size = 0;
1993         map_length = length;
1994
1995         /*
1996          * Avoid races with device replace and make sure our bbio has devices
1997          * associated to its stripes that don't go away while we are doing the
1998          * read repair operation.
1999          */
2000         btrfs_bio_counter_inc_blocked(fs_info);
2001         if (btrfs_is_parity_mirror(fs_info, logical, length)) {
2002                 /*
2003                  * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2004                  * to update all raid stripes, but here we just want to correct
2005                  * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2006                  * stripe's dev and sector.
2007                  */
2008                 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2009                                       &map_length, &bbio, 0);
2010                 if (ret) {
2011                         btrfs_bio_counter_dec(fs_info);
2012                         bio_put(bio);
2013                         return -EIO;
2014                 }
2015                 ASSERT(bbio->mirror_num == 1);
2016         } else {
2017                 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2018                                       &map_length, &bbio, mirror_num);
2019                 if (ret) {
2020                         btrfs_bio_counter_dec(fs_info);
2021                         bio_put(bio);
2022                         return -EIO;
2023                 }
2024                 BUG_ON(mirror_num != bbio->mirror_num);
2025         }
2026
2027         sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2028         bio->bi_iter.bi_sector = sector;
2029         dev = bbio->stripes[bbio->mirror_num - 1].dev;
2030         btrfs_put_bbio(bbio);
2031         if (!dev || !dev->bdev ||
2032             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2033                 btrfs_bio_counter_dec(fs_info);
2034                 bio_put(bio);
2035                 return -EIO;
2036         }
2037         bio_set_dev(bio, dev->bdev);
2038         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2039         bio_add_page(bio, page, length, pg_offset);
2040
2041         if (btrfsic_submit_bio_wait(bio)) {
2042                 /* try to remap that extent elsewhere? */
2043                 btrfs_bio_counter_dec(fs_info);
2044                 bio_put(bio);
2045                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2046                 return -EIO;
2047         }
2048
2049         btrfs_info_rl_in_rcu(fs_info,
2050                 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2051                                   ino, start,
2052                                   rcu_str_deref(dev->name), sector);
2053         btrfs_bio_counter_dec(fs_info);
2054         bio_put(bio);
2055         return 0;
2056 }
2057
2058 int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2059                          struct extent_buffer *eb, int mirror_num)
2060 {
2061         u64 start = eb->start;
2062         int i, num_pages = num_extent_pages(eb);
2063         int ret = 0;
2064
2065         if (sb_rdonly(fs_info->sb))
2066                 return -EROFS;
2067
2068         for (i = 0; i < num_pages; i++) {
2069                 struct page *p = eb->pages[i];
2070
2071                 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2072                                         start - page_offset(p), mirror_num);
2073                 if (ret)
2074                         break;
2075                 start += PAGE_SIZE;
2076         }
2077
2078         return ret;
2079 }
2080
2081 /*
2082  * each time an IO finishes, we do a fast check in the IO failure tree
2083  * to see if we need to process or clean up an io_failure_record
2084  */
2085 int clean_io_failure(struct btrfs_fs_info *fs_info,
2086                      struct extent_io_tree *failure_tree,
2087                      struct extent_io_tree *io_tree, u64 start,
2088                      struct page *page, u64 ino, unsigned int pg_offset)
2089 {
2090         u64 private;
2091         struct io_failure_record *failrec;
2092         struct extent_state *state;
2093         int num_copies;
2094         int ret;
2095
2096         private = 0;
2097         ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2098                                EXTENT_DIRTY, 0);
2099         if (!ret)
2100                 return 0;
2101
2102         ret = get_state_failrec(failure_tree, start, &failrec);
2103         if (ret)
2104                 return 0;
2105
2106         BUG_ON(!failrec->this_mirror);
2107
2108         if (failrec->in_validation) {
2109                 /* there was no real error, just free the record */
2110                 btrfs_debug(fs_info,
2111                         "clean_io_failure: freeing dummy error at %llu",
2112                         failrec->start);
2113                 goto out;
2114         }
2115         if (sb_rdonly(fs_info->sb))
2116                 goto out;
2117
2118         spin_lock(&io_tree->lock);
2119         state = find_first_extent_bit_state(io_tree,
2120                                             failrec->start,
2121                                             EXTENT_LOCKED);
2122         spin_unlock(&io_tree->lock);
2123
2124         if (state && state->start <= failrec->start &&
2125             state->end >= failrec->start + failrec->len - 1) {
2126                 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2127                                               failrec->len);
2128                 if (num_copies > 1)  {
2129                         repair_io_failure(fs_info, ino, start, failrec->len,
2130                                           failrec->logical, page, pg_offset,
2131                                           failrec->failed_mirror);
2132                 }
2133         }
2134
2135 out:
2136         free_io_failure(failure_tree, io_tree, failrec);
2137
2138         return 0;
2139 }
2140
2141 /*
2142  * Can be called when
2143  * - hold extent lock
2144  * - under ordered extent
2145  * - the inode is freeing
2146  */
2147 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2148 {
2149         struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2150         struct io_failure_record *failrec;
2151         struct extent_state *state, *next;
2152
2153         if (RB_EMPTY_ROOT(&failure_tree->state))
2154                 return;
2155
2156         spin_lock(&failure_tree->lock);
2157         state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2158         while (state) {
2159                 if (state->start > end)
2160                         break;
2161
2162                 ASSERT(state->end <= end);
2163
2164                 next = next_state(state);
2165
2166                 failrec = state->failrec;
2167                 free_extent_state(state);
2168                 kfree(failrec);
2169
2170                 state = next;
2171         }
2172         spin_unlock(&failure_tree->lock);
2173 }
2174
2175 int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2176                 struct io_failure_record **failrec_ret)
2177 {
2178         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2179         struct io_failure_record *failrec;
2180         struct extent_map *em;
2181         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2182         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2183         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2184         int ret;
2185         u64 logical;
2186
2187         ret = get_state_failrec(failure_tree, start, &failrec);
2188         if (ret) {
2189                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2190                 if (!failrec)
2191                         return -ENOMEM;
2192
2193                 failrec->start = start;
2194                 failrec->len = end - start + 1;
2195                 failrec->this_mirror = 0;
2196                 failrec->bio_flags = 0;
2197                 failrec->in_validation = 0;
2198
2199                 read_lock(&em_tree->lock);
2200                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2201                 if (!em) {
2202                         read_unlock(&em_tree->lock);
2203                         kfree(failrec);
2204                         return -EIO;
2205                 }
2206
2207                 if (em->start > start || em->start + em->len <= start) {
2208                         free_extent_map(em);
2209                         em = NULL;
2210                 }
2211                 read_unlock(&em_tree->lock);
2212                 if (!em) {
2213                         kfree(failrec);
2214                         return -EIO;
2215                 }
2216
2217                 logical = start - em->start;
2218                 logical = em->block_start + logical;
2219                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2220                         logical = em->block_start;
2221                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2222                         extent_set_compress_type(&failrec->bio_flags,
2223                                                  em->compress_type);
2224                 }
2225
2226                 btrfs_debug(fs_info,
2227                         "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2228                         logical, start, failrec->len);
2229
2230                 failrec->logical = logical;
2231                 free_extent_map(em);
2232
2233                 /* set the bits in the private failure tree */
2234                 ret = set_extent_bits(failure_tree, start, end,
2235                                         EXTENT_LOCKED | EXTENT_DIRTY);
2236                 if (ret >= 0)
2237                         ret = set_state_failrec(failure_tree, start, failrec);
2238                 /* set the bits in the inode's tree */
2239                 if (ret >= 0)
2240                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2241                 if (ret < 0) {
2242                         kfree(failrec);
2243                         return ret;
2244                 }
2245         } else {
2246                 btrfs_debug(fs_info,
2247                         "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2248                         failrec->logical, failrec->start, failrec->len,
2249                         failrec->in_validation);
2250                 /*
2251                  * when data can be on disk more than twice, add to failrec here
2252                  * (e.g. with a list for failed_mirror) to make
2253                  * clean_io_failure() clean all those errors at once.
2254                  */
2255         }
2256
2257         *failrec_ret = failrec;
2258
2259         return 0;
2260 }
2261
2262 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2263                            struct io_failure_record *failrec, int failed_mirror)
2264 {
2265         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2266         int num_copies;
2267
2268         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2269         if (num_copies == 1) {
2270                 /*
2271                  * we only have a single copy of the data, so don't bother with
2272                  * all the retry and error correction code that follows. no
2273                  * matter what the error is, it is very likely to persist.
2274                  */
2275                 btrfs_debug(fs_info,
2276                         "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2277                         num_copies, failrec->this_mirror, failed_mirror);
2278                 return false;
2279         }
2280
2281         /*
2282          * there are two premises:
2283          *      a) deliver good data to the caller
2284          *      b) correct the bad sectors on disk
2285          */
2286         if (failed_bio_pages > 1) {
2287                 /*
2288                  * to fulfill b), we need to know the exact failing sectors, as
2289                  * we don't want to rewrite any more than the failed ones. thus,
2290                  * we need separate read requests for the failed bio
2291                  *
2292                  * if the following BUG_ON triggers, our validation request got
2293                  * merged. we need separate requests for our algorithm to work.
2294                  */
2295                 BUG_ON(failrec->in_validation);
2296                 failrec->in_validation = 1;
2297                 failrec->this_mirror = failed_mirror;
2298         } else {
2299                 /*
2300                  * we're ready to fulfill a) and b) alongside. get a good copy
2301                  * of the failed sector and if we succeed, we have setup
2302                  * everything for repair_io_failure to do the rest for us.
2303                  */
2304                 if (failrec->in_validation) {
2305                         BUG_ON(failrec->this_mirror != failed_mirror);
2306                         failrec->in_validation = 0;
2307                         failrec->this_mirror = 0;
2308                 }
2309                 failrec->failed_mirror = failed_mirror;
2310                 failrec->this_mirror++;
2311                 if (failrec->this_mirror == failed_mirror)
2312                         failrec->this_mirror++;
2313         }
2314
2315         if (failrec->this_mirror > num_copies) {
2316                 btrfs_debug(fs_info,
2317                         "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2318                         num_copies, failrec->this_mirror, failed_mirror);
2319                 return false;
2320         }
2321
2322         return true;
2323 }
2324
2325
2326 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2327                                     struct io_failure_record *failrec,
2328                                     struct page *page, int pg_offset, int icsum,
2329                                     bio_end_io_t *endio_func, void *data)
2330 {
2331         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2332         struct bio *bio;
2333         struct btrfs_io_bio *btrfs_failed_bio;
2334         struct btrfs_io_bio *btrfs_bio;
2335
2336         bio = btrfs_io_bio_alloc(1);
2337         bio->bi_end_io = endio_func;
2338         bio->bi_iter.bi_sector = failrec->logical >> 9;
2339         bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
2340         bio->bi_iter.bi_size = 0;
2341         bio->bi_private = data;
2342
2343         btrfs_failed_bio = btrfs_io_bio(failed_bio);
2344         if (btrfs_failed_bio->csum) {
2345                 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2346
2347                 btrfs_bio = btrfs_io_bio(bio);
2348                 btrfs_bio->csum = btrfs_bio->csum_inline;
2349                 icsum *= csum_size;
2350                 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2351                        csum_size);
2352         }
2353
2354         bio_add_page(bio, page, failrec->len, pg_offset);
2355
2356         return bio;
2357 }
2358
2359 /*
2360  * this is a generic handler for readpage errors (default
2361  * readpage_io_failed_hook). if other copies exist, read those and write back
2362  * good data to the failed position. does not investigate in remapping the
2363  * failed extent elsewhere, hoping the device will be smart enough to do this as
2364  * needed
2365  */
2366
2367 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2368                               struct page *page, u64 start, u64 end,
2369                               int failed_mirror)
2370 {
2371         struct io_failure_record *failrec;
2372         struct inode *inode = page->mapping->host;
2373         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2374         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2375         struct bio *bio;
2376         int read_mode = 0;
2377         blk_status_t status;
2378         int ret;
2379         unsigned failed_bio_pages = bio_pages_all(failed_bio);
2380
2381         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2382
2383         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2384         if (ret)
2385                 return ret;
2386
2387         if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2388                                     failed_mirror)) {
2389                 free_io_failure(failure_tree, tree, failrec);
2390                 return -EIO;
2391         }
2392
2393         if (failed_bio_pages > 1)
2394                 read_mode |= REQ_FAILFAST_DEV;
2395
2396         phy_offset >>= inode->i_sb->s_blocksize_bits;
2397         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2398                                       start - page_offset(page),
2399                                       (int)phy_offset, failed_bio->bi_end_io,
2400                                       NULL);
2401         bio->bi_opf = REQ_OP_READ | read_mode;
2402
2403         btrfs_debug(btrfs_sb(inode->i_sb),
2404                 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2405                 read_mode, failrec->this_mirror, failrec->in_validation);
2406
2407         status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2408                                          failrec->bio_flags, 0);
2409         if (status) {
2410                 free_io_failure(failure_tree, tree, failrec);
2411                 bio_put(bio);
2412                 ret = blk_status_to_errno(status);
2413         }
2414
2415         return ret;
2416 }
2417
2418 /* lots and lots of room for performance fixes in the end_bio funcs */
2419
2420 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2421 {
2422         int uptodate = (err == 0);
2423         struct extent_io_tree *tree;
2424         int ret = 0;
2425
2426         tree = &BTRFS_I(page->mapping->host)->io_tree;
2427
2428         if (tree->ops && tree->ops->writepage_end_io_hook)
2429                 tree->ops->writepage_end_io_hook(page, start, end, NULL,
2430                                 uptodate);
2431
2432         if (!uptodate) {
2433                 ClearPageUptodate(page);
2434                 SetPageError(page);
2435                 ret = err < 0 ? err : -EIO;
2436                 mapping_set_error(page->mapping, ret);
2437         }
2438 }
2439
2440 /*
2441  * after a writepage IO is done, we need to:
2442  * clear the uptodate bits on error
2443  * clear the writeback bits in the extent tree for this IO
2444  * end_page_writeback if the page has no more pending IO
2445  *
2446  * Scheduling is not allowed, so the extent state tree is expected
2447  * to have one and only one object corresponding to this IO.
2448  */
2449 static void end_bio_extent_writepage(struct bio *bio)
2450 {
2451         int error = blk_status_to_errno(bio->bi_status);
2452         struct bio_vec *bvec;
2453         u64 start;
2454         u64 end;
2455         int i;
2456
2457         ASSERT(!bio_flagged(bio, BIO_CLONED));
2458         bio_for_each_segment_all(bvec, bio, i) {
2459                 struct page *page = bvec->bv_page;
2460                 struct inode *inode = page->mapping->host;
2461                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2462
2463                 /* We always issue full-page reads, but if some block
2464                  * in a page fails to read, blk_update_request() will
2465                  * advance bv_offset and adjust bv_len to compensate.
2466                  * Print a warning for nonzero offsets, and an error
2467                  * if they don't add up to a full page.  */
2468                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2469                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2470                                 btrfs_err(fs_info,
2471                                    "partial page write in btrfs with offset %u and length %u",
2472                                         bvec->bv_offset, bvec->bv_len);
2473                         else
2474                                 btrfs_info(fs_info,
2475                                    "incomplete page write in btrfs with offset %u and length %u",
2476                                         bvec->bv_offset, bvec->bv_len);
2477                 }
2478
2479                 start = page_offset(page);
2480                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2481
2482                 end_extent_writepage(page, error, start, end);
2483                 end_page_writeback(page);
2484         }
2485
2486         bio_put(bio);
2487 }
2488
2489 static void
2490 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2491                               int uptodate)
2492 {
2493         struct extent_state *cached = NULL;
2494         u64 end = start + len - 1;
2495
2496         if (uptodate && tree->track_uptodate)
2497                 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2498         unlock_extent_cached_atomic(tree, start, end, &cached);
2499 }
2500
2501 /*
2502  * after a readpage IO is done, we need to:
2503  * clear the uptodate bits on error
2504  * set the uptodate bits if things worked
2505  * set the page up to date if all extents in the tree are uptodate
2506  * clear the lock bit in the extent tree
2507  * unlock the page if there are no other extents locked for it
2508  *
2509  * Scheduling is not allowed, so the extent state tree is expected
2510  * to have one and only one object corresponding to this IO.
2511  */
2512 static void end_bio_extent_readpage(struct bio *bio)
2513 {
2514         struct bio_vec *bvec;
2515         int uptodate = !bio->bi_status;
2516         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2517         struct extent_io_tree *tree, *failure_tree;
2518         u64 offset = 0;
2519         u64 start;
2520         u64 end;
2521         u64 len;
2522         u64 extent_start = 0;
2523         u64 extent_len = 0;
2524         int mirror;
2525         int ret;
2526         int i;
2527
2528         ASSERT(!bio_flagged(bio, BIO_CLONED));
2529         bio_for_each_segment_all(bvec, bio, i) {
2530                 struct page *page = bvec->bv_page;
2531                 struct inode *inode = page->mapping->host;
2532                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2533
2534                 btrfs_debug(fs_info,
2535                         "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2536                         (u64)bio->bi_iter.bi_sector, bio->bi_status,
2537                         io_bio->mirror_num);
2538                 tree = &BTRFS_I(inode)->io_tree;
2539                 failure_tree = &BTRFS_I(inode)->io_failure_tree;
2540
2541                 /* We always issue full-page reads, but if some block
2542                  * in a page fails to read, blk_update_request() will
2543                  * advance bv_offset and adjust bv_len to compensate.
2544                  * Print a warning for nonzero offsets, and an error
2545                  * if they don't add up to a full page.  */
2546                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2547                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2548                                 btrfs_err(fs_info,
2549                                         "partial page read in btrfs with offset %u and length %u",
2550                                         bvec->bv_offset, bvec->bv_len);
2551                         else
2552                                 btrfs_info(fs_info,
2553                                         "incomplete page read in btrfs with offset %u and length %u",
2554                                         bvec->bv_offset, bvec->bv_len);
2555                 }
2556
2557                 start = page_offset(page);
2558                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2559                 len = bvec->bv_len;
2560
2561                 mirror = io_bio->mirror_num;
2562                 if (likely(uptodate && tree->ops)) {
2563                         ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2564                                                               page, start, end,
2565                                                               mirror);
2566                         if (ret)
2567                                 uptodate = 0;
2568                         else
2569                                 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2570                                                  failure_tree, tree, start,
2571                                                  page,
2572                                                  btrfs_ino(BTRFS_I(inode)), 0);
2573                 }
2574
2575                 if (likely(uptodate))
2576                         goto readpage_ok;
2577
2578                 if (tree->ops) {
2579                         ret = tree->ops->readpage_io_failed_hook(page, mirror);
2580                         if (ret == -EAGAIN) {
2581                                 /*
2582                                  * Data inode's readpage_io_failed_hook() always
2583                                  * returns -EAGAIN.
2584                                  *
2585                                  * The generic bio_readpage_error handles errors
2586                                  * the following way: If possible, new read
2587                                  * requests are created and submitted and will
2588                                  * end up in end_bio_extent_readpage as well (if
2589                                  * we're lucky, not in the !uptodate case). In
2590                                  * that case it returns 0 and we just go on with
2591                                  * the next page in our bio. If it can't handle
2592                                  * the error it will return -EIO and we remain
2593                                  * responsible for that page.
2594                                  */
2595                                 ret = bio_readpage_error(bio, offset, page,
2596                                                          start, end, mirror);
2597                                 if (ret == 0) {
2598                                         uptodate = !bio->bi_status;
2599                                         offset += len;
2600                                         continue;
2601                                 }
2602                         }
2603
2604                         /*
2605                          * metadata's readpage_io_failed_hook() always returns
2606                          * -EIO and fixes nothing.  -EIO is also returned if
2607                          * data inode error could not be fixed.
2608                          */
2609                         ASSERT(ret == -EIO);
2610                 }
2611 readpage_ok:
2612                 if (likely(uptodate)) {
2613                         loff_t i_size = i_size_read(inode);
2614                         pgoff_t end_index = i_size >> PAGE_SHIFT;
2615                         unsigned off;
2616
2617                         /* Zero out the end if this page straddles i_size */
2618                         off = i_size & (PAGE_SIZE-1);
2619                         if (page->index == end_index && off)
2620                                 zero_user_segment(page, off, PAGE_SIZE);
2621                         SetPageUptodate(page);
2622                 } else {
2623                         ClearPageUptodate(page);
2624                         SetPageError(page);
2625                 }
2626                 unlock_page(page);
2627                 offset += len;
2628
2629                 if (unlikely(!uptodate)) {
2630                         if (extent_len) {
2631                                 endio_readpage_release_extent(tree,
2632                                                               extent_start,
2633                                                               extent_len, 1);
2634                                 extent_start = 0;
2635                                 extent_len = 0;
2636                         }
2637                         endio_readpage_release_extent(tree, start,
2638                                                       end - start + 1, 0);
2639                 } else if (!extent_len) {
2640                         extent_start = start;
2641                         extent_len = end + 1 - start;
2642                 } else if (extent_start + extent_len == start) {
2643                         extent_len += end + 1 - start;
2644                 } else {
2645                         endio_readpage_release_extent(tree, extent_start,
2646                                                       extent_len, uptodate);
2647                         extent_start = start;
2648                         extent_len = end + 1 - start;
2649                 }
2650         }
2651
2652         if (extent_len)
2653                 endio_readpage_release_extent(tree, extent_start, extent_len,
2654                                               uptodate);
2655         if (io_bio->end_io)
2656                 io_bio->end_io(io_bio, blk_status_to_errno(bio->bi_status));
2657         bio_put(bio);
2658 }
2659
2660 /*
2661  * Initialize the members up to but not including 'bio'. Use after allocating a
2662  * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2663  * 'bio' because use of __GFP_ZERO is not supported.
2664  */
2665 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
2666 {
2667         memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2668 }
2669
2670 /*
2671  * The following helpers allocate a bio. As it's backed by a bioset, it'll
2672  * never fail.  We're returning a bio right now but you can call btrfs_io_bio
2673  * for the appropriate container_of magic
2674  */
2675 struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
2676 {
2677         struct bio *bio;
2678
2679         bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
2680         bio_set_dev(bio, bdev);
2681         bio->bi_iter.bi_sector = first_byte >> 9;
2682         btrfs_io_bio_init(btrfs_io_bio(bio));
2683         return bio;
2684 }
2685
2686 struct bio *btrfs_bio_clone(struct bio *bio)
2687 {
2688         struct btrfs_io_bio *btrfs_bio;
2689         struct bio *new;
2690
2691         /* Bio allocation backed by a bioset does not fail */
2692         new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
2693         btrfs_bio = btrfs_io_bio(new);
2694         btrfs_io_bio_init(btrfs_bio);
2695         btrfs_bio->iter = bio->bi_iter;
2696         return new;
2697 }
2698
2699 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
2700 {
2701         struct bio *bio;
2702
2703         /* Bio allocation backed by a bioset does not fail */
2704         bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
2705         btrfs_io_bio_init(btrfs_io_bio(bio));
2706         return bio;
2707 }
2708
2709 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
2710 {
2711         struct bio *bio;
2712         struct btrfs_io_bio *btrfs_bio;
2713
2714         /* this will never fail when it's backed by a bioset */
2715         bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
2716         ASSERT(bio);
2717
2718         btrfs_bio = btrfs_io_bio(bio);
2719         btrfs_io_bio_init(btrfs_bio);
2720
2721         bio_trim(bio, offset >> 9, size >> 9);
2722         btrfs_bio->iter = bio->bi_iter;
2723         return bio;
2724 }
2725
2726 static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
2727                                        unsigned long bio_flags)
2728 {
2729         blk_status_t ret = 0;
2730         struct bio_vec *bvec = bio_last_bvec_all(bio);
2731         struct page *page = bvec->bv_page;
2732         struct extent_io_tree *tree = bio->bi_private;
2733         u64 start;
2734
2735         start = page_offset(page) + bvec->bv_offset;
2736
2737         bio->bi_private = NULL;
2738
2739         if (tree->ops)
2740                 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
2741                                            mirror_num, bio_flags, start);
2742         else
2743                 btrfsic_submit_bio(bio);
2744
2745         return blk_status_to_errno(ret);
2746 }
2747
2748 /*
2749  * @opf:        bio REQ_OP_* and REQ_* flags as one value
2750  * @tree:       tree so we can call our merge_bio hook
2751  * @wbc:        optional writeback control for io accounting
2752  * @page:       page to add to the bio
2753  * @pg_offset:  offset of the new bio or to check whether we are adding
2754  *              a contiguous page to the previous one
2755  * @size:       portion of page that we want to write
2756  * @offset:     starting offset in the page
2757  * @bdev:       attach newly created bios to this bdev
2758  * @bio_ret:    must be valid pointer, newly allocated bio will be stored there
2759  * @end_io_func:     end_io callback for new bio
2760  * @mirror_num:      desired mirror to read/write
2761  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
2762  * @bio_flags:  flags of the current bio to see if we can merge them
2763  */
2764 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2765                               struct writeback_control *wbc,
2766                               struct page *page, u64 offset,
2767                               size_t size, unsigned long pg_offset,
2768                               struct block_device *bdev,
2769                               struct bio **bio_ret,
2770                               bio_end_io_t end_io_func,
2771                               int mirror_num,
2772                               unsigned long prev_bio_flags,
2773                               unsigned long bio_flags,
2774                               bool force_bio_submit)
2775 {
2776         int ret = 0;
2777         struct bio *bio;
2778         size_t page_size = min_t(size_t, size, PAGE_SIZE);
2779         sector_t sector = offset >> 9;
2780
2781         ASSERT(bio_ret);
2782
2783         if (*bio_ret) {
2784                 bool contig;
2785                 bool can_merge = true;
2786
2787                 bio = *bio_ret;
2788                 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2789                         contig = bio->bi_iter.bi_sector == sector;
2790                 else
2791                         contig = bio_end_sector(bio) == sector;
2792
2793                 if (tree->ops && tree->ops->merge_bio_hook(page, offset,
2794                                         page_size, bio, bio_flags))
2795                         can_merge = false;
2796
2797                 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2798                     force_bio_submit ||
2799                     bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2800                         ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2801                         if (ret < 0) {
2802                                 *bio_ret = NULL;
2803                                 return ret;
2804                         }
2805                         bio = NULL;
2806                 } else {
2807                         if (wbc)
2808                                 wbc_account_io(wbc, page, page_size);
2809                         return 0;
2810                 }
2811         }
2812
2813         bio = btrfs_bio_alloc(bdev, offset);
2814         bio_add_page(bio, page, page_size, pg_offset);
2815         bio->bi_end_io = end_io_func;
2816         bio->bi_private = tree;
2817         bio->bi_write_hint = page->mapping->host->i_write_hint;
2818         bio->bi_opf = opf;
2819         if (wbc) {
2820                 wbc_init_bio(wbc, bio);
2821                 wbc_account_io(wbc, page, page_size);
2822         }
2823
2824         *bio_ret = bio;
2825
2826         return ret;
2827 }
2828
2829 static void attach_extent_buffer_page(struct extent_buffer *eb,
2830                                       struct page *page)
2831 {
2832         if (!PagePrivate(page)) {
2833                 SetPagePrivate(page);
2834                 get_page(page);
2835                 set_page_private(page, (unsigned long)eb);
2836         } else {
2837                 WARN_ON(page->private != (unsigned long)eb);
2838         }
2839 }
2840
2841 void set_page_extent_mapped(struct page *page)
2842 {
2843         if (!PagePrivate(page)) {
2844                 SetPagePrivate(page);
2845                 get_page(page);
2846                 set_page_private(page, EXTENT_PAGE_PRIVATE);
2847         }
2848 }
2849
2850 static struct extent_map *
2851 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2852                  u64 start, u64 len, get_extent_t *get_extent,
2853                  struct extent_map **em_cached)
2854 {
2855         struct extent_map *em;
2856
2857         if (em_cached && *em_cached) {
2858                 em = *em_cached;
2859                 if (extent_map_in_tree(em) && start >= em->start &&
2860                     start < extent_map_end(em)) {
2861                         refcount_inc(&em->refs);
2862                         return em;
2863                 }
2864
2865                 free_extent_map(em);
2866                 *em_cached = NULL;
2867         }
2868
2869         em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
2870         if (em_cached && !IS_ERR_OR_NULL(em)) {
2871                 BUG_ON(*em_cached);
2872                 refcount_inc(&em->refs);
2873                 *em_cached = em;
2874         }
2875         return em;
2876 }
2877 /*
2878  * basic readpage implementation.  Locked extent state structs are inserted
2879  * into the tree that are removed when the IO is done (by the end_io
2880  * handlers)
2881  * XXX JDM: This needs looking at to ensure proper page locking
2882  * return 0 on success, otherwise return error
2883  */
2884 static int __do_readpage(struct extent_io_tree *tree,
2885                          struct page *page,
2886                          get_extent_t *get_extent,
2887                          struct extent_map **em_cached,
2888                          struct bio **bio, int mirror_num,
2889                          unsigned long *bio_flags, unsigned int read_flags,
2890                          u64 *prev_em_start)
2891 {
2892         struct inode *inode = page->mapping->host;
2893         u64 start = page_offset(page);
2894         const u64 end = start + PAGE_SIZE - 1;
2895         u64 cur = start;
2896         u64 extent_offset;
2897         u64 last_byte = i_size_read(inode);
2898         u64 block_start;
2899         u64 cur_end;
2900         struct extent_map *em;
2901         struct block_device *bdev;
2902         int ret = 0;
2903         int nr = 0;
2904         size_t pg_offset = 0;
2905         size_t iosize;
2906         size_t disk_io_size;
2907         size_t blocksize = inode->i_sb->s_blocksize;
2908         unsigned long this_bio_flag = 0;
2909
2910         set_page_extent_mapped(page);
2911
2912         if (!PageUptodate(page)) {
2913                 if (cleancache_get_page(page) == 0) {
2914                         BUG_ON(blocksize != PAGE_SIZE);
2915                         unlock_extent(tree, start, end);
2916                         goto out;
2917                 }
2918         }
2919
2920         if (page->index == last_byte >> PAGE_SHIFT) {
2921                 char *userpage;
2922                 size_t zero_offset = last_byte & (PAGE_SIZE - 1);
2923
2924                 if (zero_offset) {
2925                         iosize = PAGE_SIZE - zero_offset;
2926                         userpage = kmap_atomic(page);
2927                         memset(userpage + zero_offset, 0, iosize);
2928                         flush_dcache_page(page);
2929                         kunmap_atomic(userpage);
2930                 }
2931         }
2932         while (cur <= end) {
2933                 bool force_bio_submit = false;
2934                 u64 offset;
2935
2936                 if (cur >= last_byte) {
2937                         char *userpage;
2938                         struct extent_state *cached = NULL;
2939
2940                         iosize = PAGE_SIZE - pg_offset;
2941                         userpage = kmap_atomic(page);
2942                         memset(userpage + pg_offset, 0, iosize);
2943                         flush_dcache_page(page);
2944                         kunmap_atomic(userpage);
2945                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2946                                             &cached, GFP_NOFS);
2947                         unlock_extent_cached(tree, cur,
2948                                              cur + iosize - 1, &cached);
2949                         break;
2950                 }
2951                 em = __get_extent_map(inode, page, pg_offset, cur,
2952                                       end - cur + 1, get_extent, em_cached);
2953                 if (IS_ERR_OR_NULL(em)) {
2954                         SetPageError(page);
2955                         unlock_extent(tree, cur, end);
2956                         break;
2957                 }
2958                 extent_offset = cur - em->start;
2959                 BUG_ON(extent_map_end(em) <= cur);
2960                 BUG_ON(end < cur);
2961
2962                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2963                         this_bio_flag |= EXTENT_BIO_COMPRESSED;
2964                         extent_set_compress_type(&this_bio_flag,
2965                                                  em->compress_type);
2966                 }
2967
2968                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2969                 cur_end = min(extent_map_end(em) - 1, end);
2970                 iosize = ALIGN(iosize, blocksize);
2971                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2972                         disk_io_size = em->block_len;
2973                         offset = em->block_start;
2974                 } else {
2975                         offset = em->block_start + extent_offset;
2976                         disk_io_size = iosize;
2977                 }
2978                 bdev = em->bdev;
2979                 block_start = em->block_start;
2980                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2981                         block_start = EXTENT_MAP_HOLE;
2982
2983                 /*
2984                  * If we have a file range that points to a compressed extent
2985                  * and it's followed by a consecutive file range that points to
2986                  * to the same compressed extent (possibly with a different
2987                  * offset and/or length, so it either points to the whole extent
2988                  * or only part of it), we must make sure we do not submit a
2989                  * single bio to populate the pages for the 2 ranges because
2990                  * this makes the compressed extent read zero out the pages
2991                  * belonging to the 2nd range. Imagine the following scenario:
2992                  *
2993                  *  File layout
2994                  *  [0 - 8K]                     [8K - 24K]
2995                  *    |                               |
2996                  *    |                               |
2997                  * points to extent X,         points to extent X,
2998                  * offset 4K, length of 8K     offset 0, length 16K
2999                  *
3000                  * [extent X, compressed length = 4K uncompressed length = 16K]
3001                  *
3002                  * If the bio to read the compressed extent covers both ranges,
3003                  * it will decompress extent X into the pages belonging to the
3004                  * first range and then it will stop, zeroing out the remaining
3005                  * pages that belong to the other range that points to extent X.
3006                  * So here we make sure we submit 2 bios, one for the first
3007                  * range and another one for the third range. Both will target
3008                  * the same physical extent from disk, but we can't currently
3009                  * make the compressed bio endio callback populate the pages
3010                  * for both ranges because each compressed bio is tightly
3011                  * coupled with a single extent map, and each range can have
3012                  * an extent map with a different offset value relative to the
3013                  * uncompressed data of our extent and different lengths. This
3014                  * is a corner case so we prioritize correctness over
3015                  * non-optimal behavior (submitting 2 bios for the same extent).
3016                  */
3017                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3018                     prev_em_start && *prev_em_start != (u64)-1 &&
3019                     *prev_em_start != em->orig_start)
3020                         force_bio_submit = true;
3021
3022                 if (prev_em_start)
3023                         *prev_em_start = em->orig_start;
3024
3025                 free_extent_map(em);
3026                 em = NULL;
3027
3028                 /* we've found a hole, just zero and go on */
3029                 if (block_start == EXTENT_MAP_HOLE) {
3030                         char *userpage;
3031                         struct extent_state *cached = NULL;
3032
3033                         userpage = kmap_atomic(page);
3034                         memset(userpage + pg_offset, 0, iosize);
3035                         flush_dcache_page(page);
3036                         kunmap_atomic(userpage);
3037
3038                         set_extent_uptodate(tree, cur, cur + iosize - 1,
3039                                             &cached, GFP_NOFS);
3040                         unlock_extent_cached(tree, cur,
3041                                              cur + iosize - 1, &cached);
3042                         cur = cur + iosize;
3043                         pg_offset += iosize;
3044                         continue;
3045                 }
3046                 /* the get_extent function already copied into the page */
3047                 if (test_range_bit(tree, cur, cur_end,
3048                                    EXTENT_UPTODATE, 1, NULL)) {
3049                         check_page_uptodate(tree, page);
3050                         unlock_extent(tree, cur, cur + iosize - 1);
3051                         cur = cur + iosize;
3052                         pg_offset += iosize;
3053                         continue;
3054                 }
3055                 /* we have an inline extent but it didn't get marked up
3056                  * to date.  Error out
3057                  */
3058                 if (block_start == EXTENT_MAP_INLINE) {
3059                         SetPageError(page);
3060                         unlock_extent(tree, cur, cur + iosize - 1);
3061                         cur = cur + iosize;
3062                         pg_offset += iosize;
3063                         continue;
3064                 }
3065
3066                 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
3067                                          page, offset, disk_io_size,
3068                                          pg_offset, bdev, bio,
3069                                          end_bio_extent_readpage, mirror_num,
3070                                          *bio_flags,
3071                                          this_bio_flag,
3072                                          force_bio_submit);
3073                 if (!ret) {
3074                         nr++;
3075                         *bio_flags = this_bio_flag;
3076                 } else {
3077                         SetPageError(page);
3078                         unlock_extent(tree, cur, cur + iosize - 1);
3079                         goto out;
3080                 }
3081                 cur = cur + iosize;
3082                 pg_offset += iosize;
3083         }
3084 out:
3085         if (!nr) {
3086                 if (!PageError(page))
3087                         SetPageUptodate(page);
3088                 unlock_page(page);
3089         }
3090         return ret;
3091 }
3092
3093 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3094                                              struct page *pages[], int nr_pages,
3095                                              u64 start, u64 end,
3096                                              struct extent_map **em_cached,
3097                                              struct bio **bio,
3098                                              unsigned long *bio_flags,
3099                                              u64 *prev_em_start)
3100 {
3101         struct inode *inode;
3102         struct btrfs_ordered_extent *ordered;
3103         int index;
3104
3105         inode = pages[0]->mapping->host;
3106         while (1) {
3107                 lock_extent(tree, start, end);
3108                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
3109                                                      end - start + 1);
3110                 if (!ordered)
3111                         break;
3112                 unlock_extent(tree, start, end);
3113                 btrfs_start_ordered_extent(inode, ordered, 1);
3114                 btrfs_put_ordered_extent(ordered);
3115         }
3116
3117         for (index = 0; index < nr_pages; index++) {
3118                 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
3119                                 bio, 0, bio_flags, 0, prev_em_start);
3120                 put_page(pages[index]);
3121         }
3122 }
3123
3124 static void __extent_readpages(struct extent_io_t