fs/btrfs/extent_io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/bitops.h>
   4 #include <linux/slab.h>
   5 #include <linux/bio.h>
   6 #include <linux/mm.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/page-flags.h>
   9 #include <linux/spinlock.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/swap.h>
  12 #include <linux/writeback.h>
  13 #include <linux/pagevec.h>
  14 #include <linux/prefetch.h>
  15 #include <linux/cleancache.h>
  16 #include "extent_io.h"
  17 #include "extent_map.h"
  18 #include "ctree.h"
  19 #include "btrfs_inode.h"
  20 #include "volumes.h"
  21 #include "check-integrity.h"
  22 #include "locking.h"
  23 #include "rcu-string.h"
  24 #include "backref.h"
  25 #include "disk-io.h"
  26
  27 static struct kmem_cache *extent_state_cache;
  28 static struct kmem_cache *extent_buffer_cache;
  29 static struct bio_set btrfs_bioset;
  30
  31 static inline bool extent_state_in_tree(const struct extent_state *state)
  32 {
  33         return !RB_EMPTY_NODE(&state->rb_node);
  34 }
  35
  36 #ifdef CONFIG_BTRFS_DEBUG
  37 static LIST_HEAD(buffers);
  38 static LIST_HEAD(states);
  39
  40 static DEFINE_SPINLOCK(leak_lock);
  41
  42 static inline
  43 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
  44 {
  45         unsigned long flags;
  46
  47         spin_lock_irqsave(&leak_lock, flags);
  48         list_add(new, head);
  49         spin_unlock_irqrestore(&leak_lock, flags);
  50 }
  51
  52 static inline
  53 void btrfs_leak_debug_del(struct list_head *entry)
  54 {
  55         unsigned long flags;
  56
  57         spin_lock_irqsave(&leak_lock, flags);
  58         list_del(entry);
  59         spin_unlock_irqrestore(&leak_lock, flags);
  60 }
  61
  62 static inline
  63 void btrfs_leak_debug_check(void)
  64 {
  65         struct extent_state *state;
  66         struct extent_buffer *eb;
  67
  68         while (!list_empty(&states)) {
  69                 state = list_entry(states.next, struct extent_state, leak_list);
  70                 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
  71                        state->start, state->end, state->state,
  72                        extent_state_in_tree(state),
  73                        refcount_read(&state->refs));
  74                 list_del(&state->leak_list);
  75                 kmem_cache_free(extent_state_cache, state);
  76         }
  77
  78         while (!list_empty(&buffers)) {
  79                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  80                 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
  81                        eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
  82                 list_del(&eb->leak_list);
  83                 kmem_cache_free(extent_buffer_cache, eb);
  84         }
  85 }
  86
  87 #define btrfs_debug_check_extent_io_range(tree, start, end)             \
  88         __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
  89 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
  90                 struct extent_io_tree *tree, u64 start, u64 end)
  91 {
  92         struct inode *inode = tree->private_data;
  93         u64 isize;
  94
  95         if (!inode || !is_data_inode(inode))
  96                 return;
  97
  98         isize = i_size_read(inode);
  99         if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
 100                 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
 101                     "%s: ino %llu isize %llu odd range [%llu,%llu]",
 102                         caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
 103         }
 104 }
 105 #else
 106 #define btrfs_leak_debug_add(new, head) do {} while (0)
 107 #define btrfs_leak_debug_del(entry)     do {} while (0)
 108 #define btrfs_leak_debug_check()        do {} while (0)
 109 #define btrfs_debug_check_extent_io_range(c, s, e)      do {} while (0)
 110 #endif
 111
 112 struct tree_entry {
 113         u64 start;
 114         u64 end;
 115         struct rb_node rb_node;
 116 };
 117
 118 struct extent_page_data {
 119         struct bio *bio;
 120         struct extent_io_tree *tree;
 121         /* tells writepage not to lock the state bits for this range
 122          * it still does the unlocking
 123          */
 124         unsigned int extent_locked:1;
 125
 126         /* tells the submit_bio code to use REQ_SYNC */
 127         unsigned int sync_io:1;
 128 };
 129
 130 static int add_extent_changeset(struct extent_state *state, unsigned bits,
 131                                  struct extent_changeset *changeset,
 132                                  int set)
 133 {
 134         int ret;
 135
 136         if (!changeset)
 137                 return 0;
 138         if (set && (state->state & bits) == bits)
 139                 return 0;
 140         if (!set && (state->state & bits) == 0)
 141                 return 0;
 142         changeset->bytes_changed += state->end - state->start + 1;
 143         ret = ulist_add(&changeset->range_changed, state->start, state->end,
 144                         GFP_ATOMIC);
 145         return ret;
 146 }
 147
 148 static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
 149                                        unsigned long bio_flags)
 150 {
 151         blk_status_t ret = 0;
 152         struct extent_io_tree *tree = bio->bi_private;
 153
 154         bio->bi_private = NULL;
 155
 156         if (tree->ops)
 157                 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
 158                                                  mirror_num, bio_flags);
 159         else
 160                 btrfsic_submit_bio(bio);
 161
 162         return blk_status_to_errno(ret);
 163 }
 164
 165 /* Cleanup unsubmitted bios */
 166 static void end_write_bio(struct extent_page_data *epd, int ret)
 167 {
 168         if (epd->bio) {
 169                 epd->bio->bi_status = errno_to_blk_status(ret);
 170                 bio_endio(epd->bio);
 171                 epd->bio = NULL;
 172         }
 173 }
 174
 175 /*
 176  * Submit bio from extent page data via submit_one_bio
 177  *
 178  * Return 0 if everything is OK.
 179  * Return <0 for error.
 180  */
 181 static int __must_check flush_write_bio(struct extent_page_data *epd)
 182 {
 183         int ret = 0;
 184
 185         if (epd->bio) {
 186                 ret = submit_one_bio(epd->bio, 0, 0);
 187                 /*
 188                  * Clean up of epd->bio is handled by its endio function.
 189                  * And endio is either triggered by successful bio execution
 190                  * or the error handler of submit bio hook.
 191                  * So at this point, no matter what happened, we don't need
 192                  * to clean up epd->bio.
 193                  */
 194                 epd->bio = NULL;
 195         }
 196         return ret;
 197 }
 198
 199 int __init extent_io_init(void)
 200 {
 201         extent_state_cache = kmem_cache_create("btrfs_extent_state",
 202                         sizeof(struct extent_state), 0,
 203                         SLAB_MEM_SPREAD, NULL);
 204         if (!extent_state_cache)
 205                 return -ENOMEM;
 206
 207         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
 208                         sizeof(struct extent_buffer), 0,
 209                         SLAB_MEM_SPREAD, NULL);
 210         if (!extent_buffer_cache)
 211                 goto free_state_cache;
 212
 213         if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
 214                         offsetof(struct btrfs_io_bio, bio),
 215                         BIOSET_NEED_BVECS))
 216                 goto free_buffer_cache;
 217
 218         if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
 219                 goto free_bioset;
 220
 221         return 0;
 222
 223 free_bioset:
 224         bioset_exit(&btrfs_bioset);
 225
 226 free_buffer_cache:
 227         kmem_cache_destroy(extent_buffer_cache);
 228         extent_buffer_cache = NULL;
 229
 230 free_state_cache:
 231         kmem_cache_destroy(extent_state_cache);
 232         extent_state_cache = NULL;
 233         return -ENOMEM;
 234 }
 235
 236 void __cold extent_io_exit(void)
 237 {
 238         btrfs_leak_debug_check();
 239
 240         /*
 241          * Make sure all delayed rcu free are flushed before we
 242          * destroy caches.
 243          */
 244         rcu_barrier();
 245         kmem_cache_destroy(extent_state_cache);
 246         kmem_cache_destroy(extent_buffer_cache);
 247         bioset_exit(&btrfs_bioset);
 248 }
 249
 250 void extent_io_tree_init(struct btrfs_fs_info *fs_info,
 251                          struct extent_io_tree *tree, unsigned int owner,
 252                          void *private_data)
 253 {
 254         tree->fs_info = fs_info;
 255         tree->state = RB_ROOT;
 256         tree->ops = NULL;
 257         tree->dirty_bytes = 0;
 258         spin_lock_init(&tree->lock);
 259         tree->private_data = private_data;
 260         tree->owner = owner;
 261 }
 262
 263 void extent_io_tree_release(struct extent_io_tree *tree)
 264 {
 265         spin_lock(&tree->lock);
 266         /*
 267          * Do a single barrier for the waitqueue_active check here, the state
 268          * of the waitqueue should not change once extent_io_tree_release is
 269          * called.
 270          */
 271         smp_mb();
 272         while (!RB_EMPTY_ROOT(&tree->state)) {
 273                 struct rb_node *node;
 274                 struct extent_state *state;
 275
 276                 node = rb_first(&tree->state);
 277                 state = rb_entry(node, struct extent_state, rb_node);
 278                 rb_erase(&state->rb_node, &tree->state);
 279                 RB_CLEAR_NODE(&state->rb_node);
 280                 /*
 281                  * btree io trees aren't supposed to have tasks waiting for
 282                  * changes in the flags of extent states ever.
 283                  */
 284                 ASSERT(!waitqueue_active(&state->wq));
 285                 free_extent_state(state);
 286
 287                 cond_resched_lock(&tree->lock);
 288         }
 289         spin_unlock(&tree->lock);
 290 }
 291
 292 static struct extent_state *alloc_extent_state(gfp_t mask)
 293 {
 294         struct extent_state *state;
 295
 296         /*
 297          * The given mask might be not appropriate for the slab allocator,
 298          * drop the unsupported bits
 299          */
 300         mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
 301         state = kmem_cache_alloc(extent_state_cache, mask);
 302         if (!state)
 303                 return state;
 304         state->state = 0;
 305         state->failrec = NULL;
 306         RB_CLEAR_NODE(&state->rb_node);
 307         btrfs_leak_debug_add(&state->leak_list, &states);
 308         refcount_set(&state->refs, 1);
 309         init_waitqueue_head(&state->wq);
 310         trace_alloc_extent_state(state, mask, _RET_IP_);
 311         return state;
 312 }
 313
 314 void free_extent_state(struct extent_state *state)
 315 {
 316         if (!state)
 317                 return;
 318         if (refcount_dec_and_test(&state->refs)) {
 319                 WARN_ON(extent_state_in_tree(state));
 320                 btrfs_leak_debug_del(&state->leak_list);
 321                 trace_free_extent_state(state, _RET_IP_);
 322                 kmem_cache_free(extent_state_cache, state);
 323         }
 324 }
 325
 326 static struct rb_node *tree_insert(struct rb_root *root,
 327                                    struct rb_node *search_start,
 328                                    u64 offset,
 329                                    struct rb_node *node,
 330                                    struct rb_node ***p_in,
 331                                    struct rb_node **parent_in)
 332 {
 333         struct rb_node **p;
 334         struct rb_node *parent = NULL;
 335         struct tree_entry *entry;
 336
 337         if (p_in && parent_in) {
 338                 p = *p_in;
 339                 parent = *parent_in;
 340                 goto do_insert;
 341         }
 342
 343         p = search_start ? &search_start : &root->rb_node;
 344         while (*p) {
 345                 parent = *p;
 346                 entry = rb_entry(parent, struct tree_entry, rb_node);
 347
 348                 if (offset < entry->start)
 349                         p = &(*p)->rb_left;
 350                 else if (offset > entry->end)
 351                         p = &(*p)->rb_right;
 352                 else
 353                         return parent;
 354         }
 355
 356 do_insert:
 357         rb_link_node(node, parent, p);
 358         rb_insert_color(node, root);
 359         return NULL;
 360 }
 361
 362 /**
 363  * __etree_search - searche @tree for an entry that contains @offset. Such
 364  * entry would have entry->start <= offset && entry->end >= offset.
 365  *
 366  * @tree - the tree to search
 367  * @offset - offset that should fall within an entry in @tree
 368  * @next_ret - pointer to the first entry whose range ends after @offset
 369  * @prev - pointer to the first entry whose range begins before @offset
 370  * @p_ret - pointer where new node should be anchored (used when inserting an
 371  *          entry in the tree)
 372  * @parent_ret - points to entry which would have been the parent of the entry,
 373  *               containing @offset
 374  *
 375  * This function returns a pointer to the entry that contains @offset byte
 376  * address. If no such entry exists, then NULL is returned and the other
 377  * pointer arguments to the function are filled, otherwise the found entry is
 378  * returned and other pointers are left untouched.
 379  */
 380 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
 381                                       struct rb_node **next_ret,
 382                                       struct rb_node **prev_ret,
 383                                       struct rb_node ***p_ret,
 384                                       struct rb_node **parent_ret)
 385 {
 386         struct rb_root *root = &tree->state;
 387         struct rb_node **n = &root->rb_node;
 388         struct rb_node *prev = NULL;
 389         struct rb_node *orig_prev = NULL;
 390         struct tree_entry *entry;
 391         struct tree_entry *prev_entry = NULL;
 392
 393         while (*n) {
 394                 prev = *n;
 395                 entry = rb_entry(prev, struct tree_entry, rb_node);
 396                 prev_entry = entry;
 397
 398                 if (offset < entry->start)
 399                         n = &(*n)->rb_left;
 400                 else if (offset > entry->end)
 401                         n = &(*n)->rb_right;
 402                 else
 403                         return *n;
 404         }
 405
 406         if (p_ret)
 407                 *p_ret = n;
 408         if (parent_ret)
 409                 *parent_ret = prev;
 410
 411         if (next_ret) {
 412                 orig_prev = prev;
 413                 while (prev && offset > prev_entry->end) {
 414                         prev = rb_next(prev);
 415                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 416                 }
 417                 *next_ret = prev;
 418                 prev = orig_prev;
 419         }
 420
 421         if (prev_ret) {
 422                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 423                 while (prev && offset < prev_entry->start) {
 424                         prev = rb_prev(prev);
 425                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 426                 }
 427                 *prev_ret = prev;
 428         }
 429         return NULL;
 430 }
 431
 432 static inline struct rb_node *
 433 tree_search_for_insert(struct extent_io_tree *tree,
 434                        u64 offset,
 435                        struct rb_node ***p_ret,
 436                        struct rb_node **parent_ret)
 437 {
 438         struct rb_node *next= NULL;
 439         struct rb_node *ret;
 440
 441         ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
 442         if (!ret)
 443                 return next;
 444         return ret;
 445 }
 446
 447 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
 448                                           u64 offset)
 449 {
 450         return tree_search_for_insert(tree, offset, NULL, NULL);
 451 }
 452
 453 /*
 454  * utility function to look for merge candidates inside a given range.
 455  * Any extents with matching state are merged together into a single
 456  * extent in the tree.  Extents with EXTENT_IO in their state field
 457  * are not merged because the end_io handlers need to be able to do
 458  * operations on them without sleeping (or doing allocations/splits).
 459  *
 460  * This should be called with the tree lock held.
 461  */
 462 static void merge_state(struct extent_io_tree *tree,
 463                         struct extent_state *state)
 464 {
 465         struct extent_state *other;
 466         struct rb_node *other_node;
 467
 468         if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY))
 469                 return;
 470
 471         other_node = rb_prev(&state->rb_node);
 472         if (other_node) {
 473                 other = rb_entry(other_node, struct extent_state, rb_node);
 474                 if (other->end == state->start - 1 &&
 475                     other->state == state->state) {
 476                         if (tree->private_data &&
 477                             is_data_inode(tree->private_data))
 478                                 btrfs_merge_delalloc_extent(tree->private_data,
 479                                                             state, other);
 480                         state->start = other->start;
 481                         rb_erase(&other->rb_node, &tree->state);
 482                         RB_CLEAR_NODE(&other->rb_node);
 483                         free_extent_state(other);
 484                 }
 485         }
 486         other_node = rb_next(&state->rb_node);
 487         if (other_node) {
 488                 other = rb_entry(other_node, struct extent_state, rb_node);
 489                 if (other->start == state->end + 1 &&
 490                     other->state == state->state) {
 491                         if (tree->private_data &&
 492                             is_data_inode(tree->private_data))
 493                                 btrfs_merge_delalloc_extent(tree->private_data,
 494                                                             state, other);
 495                         state->end = other->end;
 496                         rb_erase(&other->rb_node, &tree->state);
 497                         RB_CLEAR_NODE(&other->rb_node);
 498                         free_extent_state(other);
 499                 }
 500         }
 501 }
 502
 503 static void set_state_bits(struct extent_io_tree *tree,
 504                            struct extent_state *state, unsigned *bits,
 505                            struct extent_changeset *changeset);
 506
 507 /*
 508  * insert an extent_state struct into the tree.  'bits' are set on the
 509  * struct before it is inserted.
 510  *
 511  * This may return -EEXIST if the extent is already there, in which case the
 512  * state struct is freed.
 513  *
 514  * The tree lock is not taken internally.  This is a utility function and
 515  * probably isn't what you want to call (see set/clear_extent_bit).
 516  */
 517 static int insert_state(struct extent_io_tree *tree,
 518                         struct extent_state *state, u64 start, u64 end,
 519                         struct rb_node ***p,
 520                         struct rb_node **parent,
 521                         unsigned *bits, struct extent_changeset *changeset)
 522 {
 523         struct rb_node *node;
 524
 525         if (end < start)
 526                 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
 527                        end, start);
 528         state->start = start;
 529         state->end = end;
 530
 531         set_state_bits(tree, state, bits, changeset);
 532
 533         node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
 534         if (node) {
 535                 struct extent_state *found;
 536                 found = rb_entry(node, struct extent_state, rb_node);
 537                 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
 538                        found->start, found->end, start, end);
 539                 return -EEXIST;
 540         }
 541         merge_state(tree, state);
 542         return 0;
 543 }
 544
 545 /*
 546  * split a given extent state struct in two, inserting the preallocated
 547  * struct 'prealloc' as the newly created second half.  'split' indicates an
 548  * offset inside 'orig' where it should be split.
 549  *
 550  * Before calling,
 551  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
 552  * are two extent state structs in the tree:
 553  * prealloc: [orig->start, split - 1]
 554  * orig: [ split, orig->end ]
 555  *
 556  * The tree locks are not taken by this function. They need to be held
 557  * by the caller.
 558  */
 559 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
 560                        struct extent_state *prealloc, u64 split)
 561 {
 562         struct rb_node *node;
 563
 564         if (tree->private_data && is_data_inode(tree->private_data))
 565                 btrfs_split_delalloc_extent(tree->private_data, orig, split);
 566
 567         prealloc->start = orig->start;
 568         prealloc->end = split - 1;
 569         prealloc->state = orig->state;
 570         orig->start = split;
 571
 572         node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
 573                            &prealloc->rb_node, NULL, NULL);
 574         if (node) {
 575                 free_extent_state(prealloc);
 576                 return -EEXIST;
 577         }
 578         return 0;
 579 }
 580
 581 static struct extent_state *next_state(struct extent_state *state)
 582 {
 583         struct rb_node *next = rb_next(&state->rb_node);
 584         if (next)
 585                 return rb_entry(next, struct extent_state, rb_node);
 586         else
 587                 return NULL;
 588 }
 589
 590 /*
 591  * utility function to clear some bits in an extent state struct.
 592  * it will optionally wake up anyone waiting on this state (wake == 1).
 593  *
 594  * If no bits are set on the state struct after clearing things, the
 595  * struct is freed and removed from the tree
 596  */
 597 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
 598                                             struct extent_state *state,
 599                                             unsigned *bits, int wake,
 600                                             struct extent_changeset *changeset)
 601 {
 602         struct extent_state *next;
 603         unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
 604         int ret;
 605
 606         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
 607                 u64 range = state->end - state->start + 1;
 608                 WARN_ON(range > tree->dirty_bytes);
 609                 tree->dirty_bytes -= range;
 610         }
 611
 612         if (tree->private_data && is_data_inode(tree->private_data))
 613                 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
 614
 615         ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
 616         BUG_ON(ret < 0);
 617         state->state &= ~bits_to_clear;
 618         if (wake)
 619                 wake_up(&state->wq);
 620         if (state->state == 0) {
 621                 next = next_state(state);
 622                 if (extent_state_in_tree(state)) {
 623                         rb_erase(&state->rb_node, &tree->state);
 624                         RB_CLEAR_NODE(&state->rb_node);
 625                         free_extent_state(state);
 626                 } else {
 627                         WARN_ON(1);
 628                 }
 629         } else {
 630                 merge_state(tree, state);
 631                 next = next_state(state);
 632         }
 633         return next;
 634 }
 635
 636 static struct extent_state *
 637 alloc_extent_state_atomic(struct extent_state *prealloc)
 638 {
 639         if (!prealloc)
 640                 prealloc = alloc_extent_state(GFP_ATOMIC);
 641
 642         return prealloc;
 643 }
 644
 645 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
 646 {
 647         struct inode *inode = tree->private_data;
 648
 649         btrfs_panic(btrfs_sb(inode->i_sb), err,
 650         "locking error: extent tree was modified by another thread while locked");
 651 }
 652
 653 /*
 654  * clear some bits on a range in the tree.  This may require splitting
 655  * or inserting elements in the tree, so the gfp mask is used to
 656  * indicate which allocations or sleeping are allowed.
 657  *
 658  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
 659  * the given range from the tree regardless of state (ie for truncate).
 660  *
 661  * the range [start, end] is inclusive.
 662  *
 663  * This takes the tree lock, and returns 0 on success and < 0 on error.
 664  */
 665 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 666                               unsigned bits, int wake, int delete,
 667                               struct extent_state **cached_state,
 668                               gfp_t mask, struct extent_changeset *changeset)
 669 {
 670         struct extent_state *state;
 671         struct extent_state *cached;
 672         struct extent_state *prealloc = NULL;
 673         struct rb_node *node;
 674         u64 last_end;
 675         int err;
 676         int clear = 0;
 677
 678         btrfs_debug_check_extent_io_range(tree, start, end);
 679         trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits);
 680
 681         if (bits & EXTENT_DELALLOC)
 682                 bits |= EXTENT_NORESERVE;
 683
 684         if (delete)
 685                 bits |= ~EXTENT_CTLBITS;
 686
 687         if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY))
 688                 clear = 1;
 689 again:
 690         if (!prealloc && gfpflags_allow_blocking(mask)) {
 691                 /*
 692                  * Don't care for allocation failure here because we might end
 693                  * up not needing the pre-allocated extent state at all, which
 694                  * is the case if we only have in the tree extent states that
 695                  * cover our input range and don't cover too any other range.
 696                  * If we end up needing a new extent state we allocate it later.
 697                  */
 698                 prealloc = alloc_extent_state(mask);
 699         }
 700
 701         spin_lock(&tree->lock);
 702         if (cached_state) {
 703                 cached = *cached_state;
 704
 705                 if (clear) {
 706                         *cached_state = NULL;
 707                         cached_state = NULL;
 708                 }
 709
 710                 if (cached && extent_state_in_tree(cached) &&
 711                     cached->start <= start && cached->end > start) {
 712                         if (clear)
 713                                 refcount_dec(&cached->refs);
 714                         state = cached;
 715                         goto hit_next;
 716                 }
 717                 if (clear)
 718                         free_extent_state(cached);
 719         }
 720         /*
 721          * this search will find the extents that end after
 722          * our range starts
 723          */
 724         node = tree_search(tree, start);
 725         if (!node)
 726                 goto out;
 727         state = rb_entry(node, struct extent_state, rb_node);
 728 hit_next:
 729         if (state->start > end)
 730                 goto out;
 731         WARN_ON(state->end < start);
 732         last_end = state->end;
 733
 734         /* the state doesn't have the wanted bits, go ahead */
 735         if (!(state->state & bits)) {
 736                 state = next_state(state);
 737                 goto next;
 738         }
 739
 740         /*
 741          *     | ---- desired range ---- |
 742          *  | state | or
 743          *  | ------------- state -------------- |
 744          *
 745          * We need to split the extent we found, and may flip
 746          * bits on second half.
 747          *
 748          * If the extent we found extends past our range, we
 749          * just split and search again.  It'll get split again
 750          * the next time though.
 751          *
 752          * If the extent we found is inside our range, we clear
 753          * the desired bit on it.
 754          */
 755
 756         if (state->start < start) {
 757                 prealloc = alloc_extent_state_atomic(prealloc);
 758                 BUG_ON(!prealloc);
 759                 err = split_state(tree, state, prealloc, start);
 760                 if (err)
 761                         extent_io_tree_panic(tree, err);
 762
 763                 prealloc = NULL;
 764                 if (err)
 765                         goto out;
 766                 if (state->end <= end) {
 767                         state = clear_state_bit(tree, state, &bits, wake,
 768                                                 changeset);
 769                         goto next;
 770                 }
 771                 goto search_again;
 772         }
 773         /*
 774          * | ---- desired range ---- |
 775          *                        | state |
 776          * We need to split the extent, and clear the bit
 777          * on the first half
 778          */
 779         if (state->start <= end && state->end > end) {
 780                 prealloc = alloc_extent_state_atomic(prealloc);
 781                 BUG_ON(!prealloc);
 782                 err = split_state(tree, state, prealloc, end + 1);
 783                 if (err)
 784                         extent_io_tree_panic(tree, err);
 785
 786                 if (wake)
 787                         wake_up(&state->wq);
 788
 789                 clear_state_bit(tree, prealloc, &bits, wake, changeset);
 790
 791                 prealloc = NULL;
 792                 goto out;
 793         }
 794
 795         state = clear_state_bit(tree, state, &bits, wake, changeset);
 796 next:
 797         if (last_end == (u64)-1)
 798                 goto out;
 799         start = last_end + 1;
 800         if (start <= end && state && !need_resched())
 801                 goto hit_next;
 802
 803 search_again:
 804         if (start > end)
 805                 goto out;
 806         spin_unlock(&tree->lock);
 807         if (gfpflags_allow_blocking(mask))
 808                 cond_resched();
 809         goto again;
 810
 811 out:
 812         spin_unlock(&tree->lock);
 813         if (prealloc)
 814                 free_extent_state(prealloc);
 815
 816         return 0;
 817
 818 }
 819
 820 static void wait_on_state(struct extent_io_tree *tree,
 821                           struct extent_state *state)
 822                 __releases(tree->lock)
 823                 __acquires(tree->lock)
 824 {
 825         DEFINE_WAIT(wait);
 826         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
 827         spin_unlock(&tree->lock);
 828         schedule();
 829         spin_lock(&tree->lock);
 830         finish_wait(&state->wq, &wait);
 831 }
 832
 833 /*
 834  * waits for one or more bits to clear on a range in the state tree.
 835  * The range [start, end] is inclusive.
 836  * The tree lock is taken by this function
 837  */
 838 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 839                             unsigned long bits)
 840 {
 841         struct extent_state *state;
 842         struct rb_node *node;
 843
 844         btrfs_debug_check_extent_io_range(tree, start, end);
 845
 846         spin_lock(&tree->lock);
 847 again:
 848         while (1) {
 849                 /*
 850                  * this search will find all the extents that end after
 851                  * our range starts
 852                  */
 853                 node = tree_search(tree, start);
 854 process_node:
 855                 if (!node)
 856                         break;
 857
 858                 state = rb_entry(node, struct extent_state, rb_node);
 859
 860                 if (state->start > end)
 861                         goto out;
 862
 863                 if (state->state & bits) {
 864                         start = state->start;
 865                         refcount_inc(&state->refs);
 866                         wait_on_state(tree, state);
 867                         free_extent_state(state);
 868                         goto again;
 869                 }
 870                 start = state->end + 1;
 871
 872                 if (start > end)
 873                         break;
 874
 875                 if (!cond_resched_lock(&tree->lock)) {
 876                         node = rb_next(node);
 877                         goto process_node;
 878                 }
 879         }
 880 out:
 881         spin_unlock(&tree->lock);
 882 }
 883
 884 static void set_state_bits(struct extent_io_tree *tree,
 885                            struct extent_state *state,
 886                            unsigned *bits, struct extent_changeset *changeset)
 887 {
 888         unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
 889         int ret;
 890
 891         if (tree->private_data && is_data_inode(tree->private_data))
 892                 btrfs_set_delalloc_extent(tree->private_data, state, bits);
 893
 894         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
 895                 u64 range = state->end - state->start + 1;
 896                 tree->dirty_bytes += range;
 897         }
 898         ret = add_extent_changeset(state, bits_to_set, changeset, 1);
 899         BUG_ON(ret < 0);
 900         state->state |= bits_to_set;
 901 }
 902
 903 static void cache_state_if_flags(struct extent_state *state,
 904                                  struct extent_state **cached_ptr,
 905                                  unsigned flags)
 906 {
 907         if (cached_ptr && !(*cached_ptr)) {
 908                 if (!flags || (state->state & flags)) {
 909                         *cached_ptr = state;
 910                         refcount_inc(&state->refs);
 911                 }
 912         }
 913 }
 914
 915 static void cache_state(struct extent_state *state,
 916                         struct extent_state **cached_ptr)
 917 {
 918         return cache_state_if_flags(state, cached_ptr,
 919                                     EXTENT_LOCKED | EXTENT_BOUNDARY);
 920 }
 921
 922 /*
 923  * set some bits on a range in the tree.  This may require allocations or
 924  * sleeping, so the gfp mask is used to indicate what is allowed.
 925  *
 926  * If any of the exclusive bits are set, this will fail with -EEXIST if some
 927  * part of the range already has the desired bits set.  The start of the
 928  * existing range is returned in failed_start in this case.
 929  *
 930  * [start, end] is inclusive This takes the tree lock.
 931  */
 932
 933 static int __must_check
 934 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 935                  unsigned bits, unsigned exclusive_bits,
 936                  u64 *failed_start, struct extent_state **cached_state,
 937                  gfp_t mask, struct extent_changeset *changeset)
 938 {
 939         struct extent_state *state;
 940         struct extent_state *prealloc = NULL;
 941         struct rb_node *node;
 942         struct rb_node **p;
 943         struct rb_node *parent;
 944         int err = 0;
 945         u64 last_start;
 946         u64 last_end;
 947
 948         btrfs_debug_check_extent_io_range(tree, start, end);
 949         trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits);
 950
 951 again:
 952         if (!prealloc && gfpflags_allow_blocking(mask)) {
 953                 /*
 954                  * Don't care for allocation failure here because we might end
 955                  * up not needing the pre-allocated extent state at all, which
 956                  * is the case if we only have in the tree extent states that
 957                  * cover our input range and don't cover too any other range.
 958                  * If we end up needing a new extent state we allocate it later.
 959                  */
 960                 prealloc = alloc_extent_state(mask);
 961         }
 962
 963         spin_lock(&tree->lock);
 964         if (cached_state && *cached_state) {
 965                 state = *cached_state;
 966                 if (state->start <= start && state->end > start &&
 967                     extent_state_in_tree(state)) {
 968                         node = &state->rb_node;
 969                         goto hit_next;
 970                 }
 971         }
 972         /*
 973          * this search will find all the extents that end after
 974          * our range starts.
 975          */
 976         node = tree_search_for_insert(tree, start, &p, &parent);
 977         if (!node) {
 978                 prealloc = alloc_extent_state_atomic(prealloc);
 979                 BUG_ON(!prealloc);
 980                 err = insert_state(tree, prealloc, start, end,
 981                                    &p, &parent, &bits, changeset);
 982                 if (err)
 983                         extent_io_tree_panic(tree, err);
 984
 985                 cache_state(prealloc, cached_state);
 986                 prealloc = NULL;
 987                 goto out;
 988         }
 989         state = rb_entry(node, struct extent_state, rb_node);
 990 hit_next:
 991         last_start = state->start;
 992         last_end = state->end;
 993
 994         /*
 995          * | ---- desired range ---- |
 996          * | state |
 997          *
 998          * Just lock what we found and keep going
 999          */
1000         if (state->start == start && state->end <= end) {
1001                 if (state->state & exclusive_bits) {
1002                         *failed_start = state->start;
1003                         err = -EEXIST;
1004                         goto out;
1005                 }
1006
1007                 set_state_bits(tree, state, &bits, changeset);
1008                 cache_state(state, cached_state);
1009                 merge_state(tree, state);
1010                 if (last_end == (u64)-1)
1011                         goto out;
1012                 start = last_end + 1;
1013                 state = next_state(state);
1014                 if (start < end && state && state->start == start &&
1015                     !need_resched())
1016                         goto hit_next;
1017                 goto search_again;
1018         }
1019
1020         /*
1021          *     | ---- desired range ---- |
1022          * | state |
1023          *   or
1024          * | ------------- state -------------- |
1025          *
1026          * We need to split the extent we found, and may flip bits on
1027          * second half.
1028          *
1029          * If the extent we found extends past our
1030          * range, we just split and search again.  It'll get split
1031          * again the next time though.
1032          *
1033          * If the extent we found is inside our range, we set the
1034          * desired bit on it.
1035          */
1036         if (state->start < start) {
1037                 if (state->state & exclusive_bits) {
1038                         *failed_start = start;
1039                         err = -EEXIST;
1040                         goto out;
1041                 }
1042
1043                 prealloc = alloc_extent_state_atomic(prealloc);
1044                 BUG_ON(!prealloc);
1045                 err = split_state(tree, state, prealloc, start);
1046                 if (err)
1047                         extent_io_tree_panic(tree, err);
1048
1049                 prealloc = NULL;
1050                 if (err)
1051                         goto out;
1052                 if (state->end <= end) {
1053                         set_state_bits(tree, state, &bits, changeset);
1054                         cache_state(state, cached_state);
1055                         merge_state(tree, state);
1056                         if (last_end == (u64)-1)
1057                                 goto out;
1058                         start = last_end + 1;
1059                         state = next_state(state);
1060                         if (start < end && state && state->start == start &&
1061                             !need_resched())
1062                                 goto hit_next;
1063                 }
1064                 goto search_again;
1065         }
1066         /*
1067          * | ---- desired range ---- |
1068          *     | state | or               | state |
1069          *
1070          * There's a hole, we need to insert something in it and
1071          * ignore the extent we found.
1072          */
1073         if (state->start > start) {
1074                 u64 this_end;
1075                 if (end < last_start)
1076                         this_end = end;
1077                 else
1078                         this_end = last_start - 1;
1079
1080                 prealloc = alloc_extent_state_atomic(prealloc);
1081                 BUG_ON(!prealloc);
1082
1083                 /*
1084                  * Avoid to free 'prealloc' if it can be merged with
1085                  * the later extent.
1086                  */
1087                 err = insert_state(tree, prealloc, start, this_end,
1088                                    NULL, NULL, &bits, changeset);
1089                 if (err)
1090                         extent_io_tree_panic(tree, err);
1091
1092                 cache_state(prealloc, cached_state);
1093                 prealloc = NULL;
1094                 start = this_end + 1;
1095                 goto search_again;
1096         }
1097         /*
1098          * | ---- desired range ---- |
1099          *                        | state |
1100          * We need to split the extent, and set the bit
1101          * on the first half
1102          */
1103         if (state->start <= end && state->end > end) {
1104                 if (state->state & exclusive_bits) {
1105                         *failed_start = start;
1106                         err = -EEXIST;
1107                         goto out;
1108                 }
1109
1110                 prealloc = alloc_extent_state_atomic(prealloc);
1111                 BUG_ON(!prealloc);
1112                 err = split_state(tree, state, prealloc, end + 1);
1113                 if (err)
1114                         extent_io_tree_panic(tree, err);
1115
1116                 set_state_bits(tree, prealloc, &bits, changeset);
1117                 cache_state(prealloc, cached_state);
1118                 merge_state(tree, prealloc);
1119                 prealloc = NULL;
1120                 goto out;
1121         }
1122
1123 search_again:
1124         if (start > end)
1125                 goto out;
1126         spin_unlock(&tree->lock);
1127         if (gfpflags_allow_blocking(mask))
1128                 cond_resched();
1129         goto again;
1130
1131 out:
1132         spin_unlock(&tree->lock);
1133         if (prealloc)
1134                 free_extent_state(prealloc);
1135
1136         return err;
1137
1138 }
1139
1140 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1141                    unsigned bits, u64 * failed_start,
1142                    struct extent_state **cached_state, gfp_t mask)
1143 {
1144         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1145                                 cached_state, mask, NULL);
1146 }
1147
1148
1149 /**
1150  * convert_extent_bit - convert all bits in a given range from one bit to
1151  *                      another
1152  * @tree:       the io tree to search
1153  * @start:      the start offset in bytes
1154  * @end:        the end offset in bytes (inclusive)
1155  * @bits:       the bits to set in this range
1156  * @clear_bits: the bits to clear in this range
1157  * @cached_state:       state that we're going to cache
1158  *
1159  * This will go through and set bits for the given range.  If any states exist
1160  * already in this range they are set with the given bit and cleared of the
1161  * clear_bits.  This is only meant to be used by things that are mergeable, ie
1162  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1163  * boundary bits like LOCK.
1164  *
1165  * All allocations are done with GFP_NOFS.
1166  */
1167 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1168                        unsigned bits, unsigned clear_bits,
1169                        struct extent_state **cached_state)
1170 {
1171         struct extent_state *state;
1172         struct extent_state *prealloc = NULL;
1173         struct rb_node *node;
1174         struct rb_node **p;
1175         struct rb_node *parent;
1176         int err = 0;
1177         u64 last_start;
1178         u64 last_end;
1179         bool first_iteration = true;
1180
1181         btrfs_debug_check_extent_io_range(tree, start, end);
1182         trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits,
1183                                        clear_bits);
1184
1185 again:
1186         if (!prealloc) {
1187                 /*
1188                  * Best effort, don't worry if extent state allocation fails
1189                  * here for the first iteration. We might have a cached state
1190                  * that matches exactly the target range, in which case no
1191                  * extent state allocations are needed. We'll only know this
1192                  * after locking the tree.
1193                  */
1194                 prealloc = alloc_extent_state(GFP_NOFS);
1195                 if (!prealloc && !first_iteration)
1196                         return -ENOMEM;
1197         }
1198
1199         spin_lock(&tree->lock);
1200         if (cached_state && *cached_state) {
1201                 state = *cached_state;
1202                 if (state->start <= start && state->end > start &&
1203                     extent_state_in_tree(state)) {
1204                         node = &state->rb_node;
1205                         goto hit_next;
1206                 }
1207         }
1208
1209         /*
1210          * this search will find all the extents that end after
1211          * our range starts.
1212          */
1213         node = tree_search_for_insert(tree, start, &p, &parent);
1214         if (!node) {
1215                 prealloc = alloc_extent_state_atomic(prealloc);
1216                 if (!prealloc) {
1217                         err = -ENOMEM;
1218                         goto out;
1219                 }
1220                 err = insert_state(tree, prealloc, start, end,
1221                                    &p, &parent, &bits, NULL);
1222                 if (err)
1223                         extent_io_tree_panic(tree, err);
1224                 cache_state(prealloc, cached_state);
1225                 prealloc = NULL;
1226                 goto out;
1227         }
1228         state = rb_entry(node, struct extent_state, rb_node);
1229 hit_next:
1230         last_start = state->start;
1231         last_end = state->end;
1232
1233         /*
1234          * | ---- desired range ---- |
1235          * | state |
1236          *
1237          * Just lock what we found and keep going
1238          */
1239         if (state->start == start && state->end <= end) {
1240                 set_state_bits(tree, state, &bits, NULL);
1241                 cache_state(state, cached_state);
1242                 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1243                 if (last_end == (u64)-1)
1244                         goto out;
1245                 start = last_end + 1;
1246                 if (start < end && state && state->start == start &&
1247                     !need_resched())
1248                         goto hit_next;
1249                 goto search_again;
1250         }
1251
1252         /*
1253          *     | ---- desired range ---- |
1254          * | state |
1255          *   or
1256          * | ------------- state -------------- |
1257          *
1258          * We need to split the extent we found, and may flip bits on
1259          * second half.
1260          *
1261          * If the extent we found extends past our
1262          * range, we just split and search again.  It'll get split
1263          * again the next time though.
1264          *
1265          * If the extent we found is inside our range, we set the
1266          * desired bit on it.
1267          */
1268         if (state->start < start) {
1269                 prealloc = alloc_extent_state_atomic(prealloc);
1270                 if (!prealloc) {
1271                         err = -ENOMEM;
1272                         goto out;
1273                 }
1274                 err = split_state(tree, state, prealloc, start);
1275                 if (err)
1276                         extent_io_tree_panic(tree, err);
1277                 prealloc = NULL;
1278                 if (err)
1279                         goto out;
1280                 if (state->end <= end) {
1281                         set_state_bits(tree, state, &bits, NULL);
1282                         cache_state(state, cached_state);
1283                         state = clear_state_bit(tree, state, &clear_bits, 0,
1284                                                 NULL);
1285                         if (last_end == (u64)-1)
1286                                 goto out;
1287                         start = last_end + 1;
1288                         if (start < end && state && state->start == start &&
1289                             !need_resched())
1290                                 goto hit_next;
1291                 }
1292                 goto search_again;
1293         }
1294         /*
1295          * | ---- desired range ---- |
1296          *     | state | or               | state |
1297          *
1298          * There's a hole, we need to insert something in it and
1299          * ignore the extent we found.
1300          */
1301         if (state->start > start) {
1302                 u64 this_end;
1303                 if (end < last_start)
1304                         this_end = end;
1305                 else
1306                         this_end = last_start - 1;
1307
1308                 prealloc = alloc_extent_state_atomic(prealloc);
1309                 if (!prealloc) {
1310                         err = -ENOMEM;
1311                         goto out;
1312                 }
1313
1314                 /*
1315                  * Avoid to free 'prealloc' if it can be merged with
1316                  * the later extent.
1317                  */
1318                 err = insert_state(tree, prealloc, start, this_end,
1319                                    NULL, NULL, &bits, NULL);
1320                 if (err)
1321                         extent_io_tree_panic(tree, err);
1322                 cache_state(prealloc, cached_state);
1323                 prealloc = NULL;
1324                 start = this_end + 1;
1325                 goto search_again;
1326         }
1327         /*
1328          * | ---- desired range ---- |
1329          *                        | state |
1330          * We need to split the extent, and set the bit
1331          * on the first half
1332          */
1333         if (state->start <= end && state->end > end) {
1334                 prealloc = alloc_extent_state_atomic(prealloc);
1335                 if (!prealloc) {
1336                         err = -ENOMEM;
1337                         goto out;
1338                 }
1339
1340                 err = split_state(tree, state, prealloc, end + 1);
1341                 if (err)
1342                         extent_io_tree_panic(tree, err);
1343
1344                 set_state_bits(tree, prealloc, &bits, NULL);
1345                 cache_state(prealloc, cached_state);
1346                 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1347                 prealloc = NULL;
1348                 goto out;
1349         }
1350
1351 search_again:
1352         if (start > end)
1353                 goto out;
1354         spin_unlock(&tree->lock);
1355         cond_resched();
1356         first_iteration = false;
1357         goto again;
1358
1359 out:
1360         spin_unlock(&tree->lock);
1361         if (prealloc)
1362                 free_extent_state(prealloc);
1363
1364         return err;
1365 }
1366
1367 /* wrappers around set/clear extent bit */
1368 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1369                            unsigned bits, struct extent_changeset *changeset)
1370 {
1371         /*
1372          * We don't support EXTENT_LOCKED yet, as current changeset will
1373          * record any bits changed, so for EXTENT_LOCKED case, it will
1374          * either fail with -EEXIST or changeset will record the whole
1375          * range.
1376          */
1377         BUG_ON(bits & EXTENT_LOCKED);
1378
1379         return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1380                                 changeset);
1381 }
1382
1383 int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end,
1384                            unsigned bits)
1385 {
1386         return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL,
1387                                 GFP_NOWAIT, NULL);
1388 }
1389
1390 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1391                      unsigned bits, int wake, int delete,
1392                      struct extent_state **cached)
1393 {
1394         return __clear_extent_bit(tree, start, end, bits, wake, delete,
1395                                   cached, GFP_NOFS, NULL);
1396 }
1397
1398 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1399                 unsigned bits, struct extent_changeset *changeset)
1400 {
1401         /*
1402          * Don't support EXTENT_LOCKED case, same reason as
1403          * set_record_extent_bits().
1404          */
1405         BUG_ON(bits & EXTENT_LOCKED);
1406
1407         return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1408                                   changeset);
1409 }
1410
1411 /*
1412  * either insert or lock state struct between start and end use mask to tell
1413  * us if waiting is desired.
1414  */
1415 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1416                      struct extent_state **cached_state)
1417 {
1418         int err;
1419         u64 failed_start;
1420
1421         while (1) {
1422                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1423                                        EXTENT_LOCKED, &failed_start,
1424                                        cached_state, GFP_NOFS, NULL);
1425                 if (err == -EEXIST) {
1426                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1427                         start = failed_start;
1428                 } else
1429                         break;
1430                 WARN_ON(start > end);
1431         }
1432         return err;
1433 }
1434
1435 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1436 {
1437         int err;
1438         u64 failed_start;
1439
1440         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1441                                &failed_start, NULL, GFP_NOFS, NULL);
1442         if (err == -EEXIST) {
1443                 if (failed_start > start)
1444                         clear_extent_bit(tree, start, failed_start - 1,
1445                                          EXTENT_LOCKED, 1, 0, NULL);
1446                 return 0;
1447         }
1448         return 1;
1449 }
1450
1451 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1452 {
1453         unsigned long index = start >> PAGE_SHIFT;
1454         unsigned long end_index = end >> PAGE_SHIFT;
1455         struct page *page;
1456
1457         while (index <= end_index) {
1458                 page = find_get_page(inode->i_mapping, index);
1459                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1460                 clear_page_dirty_for_io(page);
1461                 put_page(page);
1462                 index++;
1463         }
1464 }
1465
1466 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1467 {
1468         unsigned long index = start >> PAGE_SHIFT;
1469         unsigned long end_index = end >> PAGE_SHIFT;
1470         struct page *page;
1471
1472         while (index <= end_index) {
1473                 page = find_get_page(inode->i_mapping, index);
1474                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1475                 __set_page_dirty_nobuffers(page);
1476                 account_page_redirty(page);
1477                 put_page(page);
1478                 index++;
1479         }
1480 }
1481
1482 /* find the first state struct with 'bits' set after 'start', and
1483  * return it.  tree->lock must be held.  NULL will returned if
1484  * nothing was found after 'start'
1485  */
1486 static struct extent_state *
1487 find_first_extent_bit_state(struct extent_io_tree *tree,
1488                             u64 start, unsigned bits)
1489 {
1490         struct rb_node *node;
1491         struct extent_state *state;
1492
1493         /*
1494          * this search will find all the extents that end after
1495          * our range starts.
1496          */
1497         node = tree_search(tree, start);
1498         if (!node)
1499                 goto out;
1500
1501         while (1) {
1502                 state = rb_entry(node, struct extent_state, rb_node);
1503                 if (state->end >= start && (state->state & bits))
1504                         return state;
1505
1506                 node = rb_next(node);
1507                 if (!node)
1508                         break;
1509         }
1510 out:
1511         return NULL;
1512 }
1513
1514 /*
1515  * find the first offset in the io tree with 'bits' set. zero is
1516  * returned if we find something, and *start_ret and *end_ret are
1517  * set to reflect the state struct that was found.
1518  *
1519  * If nothing was found, 1 is returned. If found something, return 0.
1520  */
1521 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1522                           u64 *start_ret, u64 *end_ret, unsigned bits,
1523                           struct extent_state **cached_state)
1524 {
1525         struct extent_state *state;
1526         int ret = 1;
1527
1528         spin_lock(&tree->lock);
1529         if (cached_state && *cached_state) {
1530                 state = *cached_state;
1531                 if (state->end == start - 1 && extent_state_in_tree(state)) {
1532                         while ((state = next_state(state)) != NULL) {
1533                                 if (state->state & bits)
1534                                         goto got_it;
1535                         }
1536                         free_extent_state(*cached_state);
1537                         *cached_state = NULL;
1538                         goto out;
1539                 }
1540                 free_extent_state(*cached_state);
1541                 *cached_state = NULL;
1542         }
1543
1544         state = find_first_extent_bit_state(tree, start, bits);
1545 got_it:
1546         if (state) {
1547                 cache_state_if_flags(state, cached_state, 0);
1548                 *start_ret = state->start;
1549                 *end_ret = state->end;
1550                 ret = 0;
1551         }
1552 out:
1553         spin_unlock(&tree->lock);
1554         return ret;
1555 }
1556
1557 /**
1558  * find_first_clear_extent_bit - find the first range that has @bits not set.
1559  * This range could start before @start.
1560  *
1561  * @tree - the tree to search
1562  * @start - the offset at/after which the found extent should start
1563  * @start_ret - records the beginning of the range
1564  * @end_ret - records the end of the range (inclusive)
1565  * @bits - the set of bits which must be unset
1566  *
1567  * Since unallocated range is also considered one which doesn't have the bits
1568  * set it's possible that @end_ret contains -1, this happens in case the range
1569  * spans (last_range_end, end of device]. In this case it's up to the caller to
1570  * trim @end_ret to the appropriate size.
1571  */
1572 void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start,
1573                                  u64 *start_ret, u64 *end_ret, unsigned bits)
1574 {
1575         struct extent_state *state;
1576         struct rb_node *node, *prev = NULL, *next;
1577
1578         spin_lock(&tree->lock);
1579
1580         /* Find first extent with bits cleared */
1581         while (1) {
1582                 node = __etree_search(tree, start, &next, &prev, NULL, NULL);
1583                 if (!node) {
1584                         node = next;
1585                         if (!node) {
1586                                 /*
1587                                  * We are past the last allocated chunk,
1588                                  * set start at the end of the last extent. The
1589                                  * device alloc tree should never be empty so
1590                                  * prev is always set.
1591                                  */
1592                                 ASSERT(prev);
1593                                 state = rb_entry(prev, struct extent_state, rb_node);
1594                                 *start_ret = state->end + 1;
1595                                 *end_ret = -1;
1596                                 goto out;
1597                         }
1598                 }
1599                 /*
1600                  * At this point 'node' either contains 'start' or start is
1601                  * before 'node'
1602                  */
1603                 state = rb_entry(node, struct extent_state, rb_node);
1604
1605                 if (in_range(start, state->start, state->end - state->start + 1)) {
1606                         if (state->state & bits) {
1607                                 /*
1608                                  * |--range with bits sets--|
1609                                  *    |
1610                                  *    start
1611                                  */
1612                                 start = state->end + 1;
1613                         } else {
1614                                 /*
1615                                  * 'start' falls within a range that doesn't
1616                                  * have the bits set, so take its start as
1617                                  * the beginning of the desired range
1618                                  *
1619                                  * |--range with bits cleared----|
1620                                  *      |
1621                                  *      start
1622                                  */
1623                                 *start_ret = state->start;
1624                                 break;
1625                         }
1626                 } else {
1627                         /*
1628                          * |---prev range---|---hole/unset---|---node range---|
1629                          *                          |
1630                          *                        start
1631                          *
1632                          *                        or
1633                          *
1634                          * |---hole/unset--||--first node--|
1635                          * 0   |
1636                          *    start
1637                          */
1638                         if (prev) {
1639                                 state = rb_entry(prev, struct extent_state,
1640                                                  rb_node);
1641                                 *start_ret = state->end + 1;
1642                         } else {
1643                                 *start_ret = 0;
1644                         }
1645                         break;
1646                 }
1647         }
1648
1649         /*
1650          * Find the longest stretch from start until an entry which has the
1651          * bits set
1652          */
1653         while (1) {
1654                 state = rb_entry(node, struct extent_state, rb_node);
1655                 if (state->end >= start && !(state->state & bits)) {
1656                         *end_ret = state->end;
1657                 } else {
1658                         *end_ret = state->start - 1;
1659                         break;
1660                 }
1661
1662                 node = rb_next(node);
1663                 if (!node)
1664                         break;
1665         }
1666 out:
1667         spin_unlock(&tree->lock);
1668 }
1669
1670 /*
1671  * find a contiguous range of bytes in the file marked as delalloc, not
1672  * more than 'max_bytes'.  start and end are used to return the range,
1673  *
1674  * true is returned if we find something, false if nothing was in the tree
1675  */
1676 static noinline bool find_delalloc_range(struct extent_io_tree *tree,
1677                                         u64 *start, u64 *end, u64 max_bytes,
1678                                         struct extent_state **cached_state)
1679 {
1680         struct rb_node *node;
1681         struct extent_state *state;
1682         u64 cur_start = *start;
1683         bool found = false;
1684         u64 total_bytes = 0;
1685
1686         spin_lock(&tree->lock);
1687
1688         /*
1689          * this search will find all the extents that end after
1690          * our range starts.
1691          */
1692         node = tree_search(tree, cur_start);
1693         if (!node) {
1694                 *end = (u64)-1;
1695                 goto out;
1696         }
1697
1698         while (1) {
1699                 state = rb_entry(node, struct extent_state, rb_node);
1700                 if (found && (state->start != cur_start ||
1701                               (state->state & EXTENT_BOUNDARY))) {
1702                         goto out;
1703                 }
1704                 if (!(state->state & EXTENT_DELALLOC)) {
1705                         if (!found)
1706                                 *end = state->end;
1707                         goto out;
1708                 }
1709                 if (!found) {
1710                         *start = state->start;
1711                         *cached_state = state;
1712                         refcount_inc(&state->refs);
1713                 }
1714                 found = true;
1715                 *end = state->end;
1716                 cur_start = state->end + 1;
1717                 node = rb_next(node);
1718                 total_bytes += state->end - state->start + 1;
1719                 if (total_bytes >= max_bytes)
1720                         break;
1721                 if (!node)
1722                         break;
1723         }
1724 out:
1725         spin_unlock(&tree->lock);
1726         return found;
1727 }
1728
1729 static int __process_pages_contig(struct address_space *mapping,
1730                                   struct page *locked_page,
1731                                   pgoff_t start_index, pgoff_t end_index,
1732                                   unsigned long page_ops, pgoff_t *index_ret);
1733
1734 static noinline void __unlock_for_delalloc(struct inode *inode,
1735                                            struct page *locked_page,
1736                                            u64 start, u64 end)
1737 {
1738         unsigned long index = start >> PAGE_SHIFT;
1739         unsigned long end_index = end >> PAGE_SHIFT;
1740
1741         ASSERT(locked_page);
1742         if (index == locked_page->index && end_index == index)
1743                 return;
1744
1745         __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1746                                PAGE_UNLOCK, NULL);
1747 }
1748
1749 static noinline int lock_delalloc_pages(struct inode *inode,
1750                                         struct page *locked_page,
1751                                         u64 delalloc_start,
1752                                         u64 delalloc_end)
1753 {
1754         unsigned long index = delalloc_start >> PAGE_SHIFT;
1755         unsigned long index_ret = index;
1756         unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1757         int ret;
1758
1759         ASSERT(locked_page);
1760         if (index == locked_page->index && index == end_index)
1761                 return 0;
1762
1763         ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1764                                      end_index, PAGE_LOCK, &index_ret);
1765         if (ret == -EAGAIN)
1766                 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1767                                       (u64)index_ret << PAGE_SHIFT);
1768         return ret;
1769 }
1770
1771 /*
1772  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1773  * more than @max_bytes.  @Start and @end are used to return the range,
1774  *
1775  * Return: true if we find something
1776  *         false if nothing was in the tree
1777  */
1778 EXPORT_FOR_TESTS
1779 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
1780                                     struct extent_io_tree *tree,
1781                                     struct page *locked_page, u64 *start,
1782                                     u64 *end)
1783 {
1784         u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
1785         u64 delalloc_start;
1786         u64 delalloc_end;
1787         bool found;
1788         struct extent_state *cached_state = NULL;
1789         int ret;
1790         int loops = 0;
1791
1792 again:
1793         /* step one, find a bunch of delalloc bytes starting at start */
1794         delalloc_start = *start;
1795         delalloc_end = 0;
1796         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1797                                     max_bytes, &cached_state);
1798         if (!found || delalloc_end <= *start) {
1799                 *start = delalloc_start;
1800                 *end = delalloc_end;
1801                 free_extent_state(cached_state);
1802                 return false;
1803         }
1804
1805         /*
1806          * start comes from the offset of locked_page.  We have to lock
1807          * pages in order, so we can't process delalloc bytes before
1808          * locked_page
1809          */
1810         if (delalloc_start < *start)
1811                 delalloc_start = *start;
1812
1813         /*
1814          * make sure to limit the number of pages we try to lock down
1815          */
1816         if (delalloc_end + 1 - delalloc_start > max_bytes)
1817                 delalloc_end = delalloc_start + max_bytes - 1;
1818
1819         /* step two, lock all the pages after the page that has start */
1820         ret = lock_delalloc_pages(inode, locked_page,
1821                                   delalloc_start, delalloc_end);
1822         ASSERT(!ret || ret == -EAGAIN);
1823         if (ret == -EAGAIN) {
1824                 /* some of the pages are gone, lets avoid looping by
1825                  * shortening the size of the delalloc range we're searching
1826                  */
1827                 free_extent_state(cached_state);
1828                 cached_state = NULL;
1829                 if (!loops) {
1830                         max_bytes = PAGE_SIZE;
1831                         loops = 1;
1832                         goto again;
1833                 } else {
1834                         found = false;
1835                         goto out_failed;
1836                 }
1837         }
1838
1839         /* step three, lock the state bits for the whole range */
1840         lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1841
1842         /* then test to make sure it is all still delalloc */
1843         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1844                              EXTENT_DELALLOC, 1, cached_state);
1845         if (!ret) {
1846                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1847                                      &cached_state);
1848                 __unlock_for_delalloc(inode, locked_page,
1849                               delalloc_start, delalloc_end);
1850                 cond_resched();
1851                 goto again;
1852         }
1853         free_extent_state(cached_state);
1854         *start = delalloc_start;
1855         *end = delalloc_end;
1856 out_failed:
1857         return found;
1858 }
1859
1860 static int __process_pages_contig(struct address_space *mapping,
1861                                   struct page *locked_page,
1862                                   pgoff_t start_index, pgoff_t end_index,
1863                                   unsigned long page_ops, pgoff_t *index_ret)
1864 {
1865         unsigned long nr_pages = end_index - start_index + 1;
1866         unsigned long pages_locked = 0;
1867         pgoff_t index = start_index;
1868         struct page *pages[16];
1869         unsigned ret;
1870         int err = 0;
1871         int i;
1872
1873         if (page_ops & PAGE_LOCK) {
1874                 ASSERT(page_ops == PAGE_LOCK);
1875                 ASSERT(index_ret && *index_ret == start_index);
1876         }
1877
1878         if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1879                 mapping_set_error(mapping, -EIO);
1880
1881         while (nr_pages > 0) {
1882                 ret = find_get_pages_contig(mapping, index,
1883                                      min_t(unsigned long,
1884                                      nr_pages, ARRAY_SIZE(pages)), pages);
1885                 if (ret == 0) {
1886                         /*
1887                          * Only if we're going to lock these pages,
1888                          * can we find nothing at @index.
1889                          */
1890                         ASSERT(page_ops & PAGE_LOCK);
1891                         err = -EAGAIN;
1892                         goto out;
1893                 }
1894
1895                 for (i = 0; i < ret; i++) {
1896                         if (page_ops & PAGE_SET_PRIVATE2)
1897                                 SetPagePrivate2(pages[i]);
1898
1899                         if (pages[i] == locked_page) {
1900                                 put_page(pages[i]);
1901                                 pages_locked++;
1902                                 continue;
1903                         }
1904                         if (page_ops & PAGE_CLEAR_DIRTY)
1905                                 clear_page_dirty_for_io(pages[i]);
1906                         if (page_ops & PAGE_SET_WRITEBACK)
1907                                 set_page_writeback(pages[i]);
1908                         if (page_ops & PAGE_SET_ERROR)
1909                                 SetPageError(pages[i]);
1910                         if (page_ops & PAGE_END_WRITEBACK)
1911                                 end_page_writeback(pages[i]);
1912                         if (page_ops & PAGE_UNLOCK)
1913                                 unlock_page(pages[i]);
1914                         if (page_ops & PAGE_LOCK) {
1915                                 lock_page(pages[i]);
1916                                 if (!PageDirty(pages[i]) ||
1917                                     pages[i]->mapping != mapping) {
1918                                         unlock_page(pages[i]);
1919                                         put_page(pages[i]);
1920                                         err = -EAGAIN;
1921                                         goto out;
1922                                 }
1923                         }
1924                         put_page(pages[i]);
1925                         pages_locked++;
1926                 }
1927                 nr_pages -= ret;
1928                 index += ret;
1929                 cond_resched();
1930         }
1931 out:
1932         if (err && index_ret)
1933                 *index_ret = start_index + pages_locked - 1;
1934         return err;
1935 }
1936
1937 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1938                                  u64 delalloc_end, struct page *locked_page,
1939                                  unsigned clear_bits,
1940                                  unsigned long page_ops)
1941 {
1942         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
1943                          NULL);
1944
1945         __process_pages_contig(inode->i_mapping, locked_page,
1946                                start >> PAGE_SHIFT, end >> PAGE_SHIFT,
1947                                page_ops, NULL);
1948 }
1949
1950 /*
1951  * count the number of bytes in the tree that have a given bit(s)
1952  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1953  * cached.  The total number found is returned.
1954  */
1955 u64 count_range_bits(struct extent_io_tree *tree,
1956                      u64 *start, u64 search_end, u64 max_bytes,
1957                      unsigned bits, int contig)
1958 {
1959         struct rb_node *node;
1960         struct extent_state *state;
1961         u64 cur_start = *start;
1962         u64 total_bytes = 0;
1963         u64 last = 0;
1964         int found = 0;
1965
1966         if (WARN_ON(search_end <= cur_start))
1967                 return 0;
1968
1969         spin_lock(&tree->lock);
1970         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1971                 total_bytes = tree->dirty_bytes;
1972                 goto out;
1973         }
1974         /*
1975          * this search will find all the extents that end after
1976          * our range starts.
1977          */
1978         node = tree_search(tree, cur_start);
1979         if (!node)
1980                 goto out;
1981
1982         while (1) {
1983                 state = rb_entry(node, struct extent_state, rb_node);
1984                 if (state->start > search_end)
1985                         break;
1986                 if (contig && found && state->start > last + 1)
1987                         break;
1988                 if (state->end >= cur_start && (state->state & bits) == bits) {
1989                         total_bytes += min(search_end, state->end) + 1 -
1990                                        max(cur_start, state->start);
1991                         if (total_bytes >= max_bytes)
1992                                 break;
1993                         if (!found) {
1994                                 *start = max(cur_start, state->start);
1995                                 found = 1;
1996                         }
1997                         last = state->end;
1998                 } else if (contig && found) {
1999                         break;
2000                 }
2001                 node = rb_next(node);
2002                 if (!node)
2003                         break;
2004         }
2005 out:
2006         spin_unlock(&tree->lock);
2007         return total_bytes;
2008 }
2009
2010 /*
2011  * set the private field for a given byte offset in the tree.  If there isn't
2012  * an extent_state there already, this does nothing.
2013  */
2014 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
2015                 struct io_failure_record *failrec)
2016 {
2017         struct rb_node *node;
2018         struct extent_state *state;
2019         int ret = 0;
2020
2021         spin_lock(&tree->lock);
2022         /*
2023          * this search will find all the extents that end after
2024          * our range starts.
2025          */
2026         node = tree_search(tree, start);
2027         if (!node) {
2028                 ret = -ENOENT;
2029                 goto out;
2030         }
2031         state = rb_entry(node, struct extent_state, rb_node);
2032         if (state->start != start) {
2033                 ret = -ENOENT;
2034                 goto out;
2035         }
2036         state->failrec = failrec;
2037 out:
2038         spin_unlock(&tree->lock);
2039         return ret;
2040 }
2041
2042 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
2043                 struct io_failure_record **failrec)
2044 {
2045         struct rb_node *node;
2046         struct extent_state *state;
2047         int ret = 0;
2048
2049         spin_lock(&tree->lock);
2050         /*
2051          * this search will find all the extents that end after
2052          * our range starts.
2053          */
2054         node = tree_search(tree, start);
2055         if (!node) {
2056                 ret = -ENOENT;
2057                 goto out;
2058         }
2059         state = rb_entry(node, struct extent_state, rb_node);
2060         if (state->start != start) {
2061                 ret = -ENOENT;
2062                 goto out;
2063         }
2064         *failrec = state->failrec;
2065 out:
2066         spin_unlock(&tree->lock);
2067         return ret;
2068 }
2069
2070 /*
2071  * searches a range in the state tree for a given mask.
2072  * If 'filled' == 1, this returns 1 only if every extent in the tree
2073  * has the bits set.  Otherwise, 1 is returned if any bit in the
2074  * range is found set.
2075  */
2076 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
2077                    unsigned bits, int filled, struct extent_state *cached)
2078 {
2079         struct extent_state *state = NULL;
2080         struct rb_node *node;
2081         int bitset = 0;
2082
2083         spin_lock(&tree->lock);
2084         if (cached && extent_state_in_tree(cached) && cached->start <= start &&
2085             cached->end > start)
2086                 node = &cached->rb_node;
2087         else
2088                 node = tree_search(tree, start);
2089         while (node && start <= end) {
2090                 state = rb_entry(node, struct extent_state, rb_node);
2091
2092                 if (filled && state->start > start) {
2093                         bitset = 0;
2094                         break;
2095                 }
2096
2097                 if (state->start > end)
2098                         break;
2099
2100                 if (state->state & bits) {
2101                         bitset = 1;
2102                         if (!filled)
2103                                 break;
2104                 } else if (filled) {
2105                         bitset = 0;
2106                         break;
2107                 }
2108
2109                 if (state->end == (u64)-1)
2110                         break;
2111
2112                 start = state->end + 1;
2113                 if (start > end)
2114                         break;
2115                 node = rb_next(node);
2116                 if (!node) {
2117                         if (filled)
2118                                 bitset = 0;
2119                         break;
2120                 }
2121         }
2122         spin_unlock(&tree->lock);
2123         return bitset;
2124 }
2125
2126 /*
2127  * helper function to set a given page up to date if all the
2128  * extents in the tree for that page are up to date
2129  */
2130 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
2131 {
2132         u64 start = page_offset(page);
2133         u64 end = start + PAGE_SIZE - 1;
2134         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
2135                 SetPageUptodate(page);
2136 }
2137
2138 int free_io_failure(struct extent_io_tree *failure_tree,
2139                     struct extent_io_tree *io_tree,
2140                     struct io_failure_record *rec)
2141 {
2142         int ret;
2143         int err = 0;
2144
2145         set_state_failrec(failure_tree, rec->start, NULL);
2146         ret = clear_extent_bits(failure_tree, rec->start,
2147                                 rec->start + rec->len - 1,
2148                                 EXTENT_LOCKED | EXTENT_DIRTY);
2149         if (ret)
2150                 err = ret;
2151
2152         ret = clear_extent_bits(io_tree, rec->start,
2153                                 rec->start + rec->len - 1,
2154                                 EXTENT_DAMAGED);
2155         if (ret && !err)
2156                 err = ret;
2157
2158         kfree(rec);
2159         return err;
2160 }
2161
2162 /*
2163  * this bypasses the standard btrfs submit functions deliberately, as
2164  * the standard behavior is to write all copies in a raid setup. here we only
2165  * want to write the one bad copy. so we do the mapping for ourselves and issue
2166  * submit_bio directly.
2167  * to avoid any synchronization issues, wait for the data after writing, which
2168  * actually prevents the read that triggered the error from finishing.
2169  * currently, there can be no more than two copies of every data bit. thus,
2170  * exactly one rewrite is required.
2171  */
2172 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
2173                       u64 length, u64 logical, struct page *page,
2174                       unsigned int pg_offset, int mirror_num)
2175 {
2176         struct bio *bio;
2177         struct btrfs_device *dev;
2178         u64 map_length = 0;
2179         u64 sector;
2180         struct btrfs_bio *bbio = NULL;
2181         int ret;
2182
2183         ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
2184         BUG_ON(!mirror_num);
2185
2186         bio = btrfs_io_bio_alloc(1);
2187         bio->bi_iter.bi_size = 0;
2188         map_length = length;
2189
2190         /*
2191          * Avoid races with device replace and make sure our bbio has devices
2192          * associated to its stripes that don't go away while we are doing the
2193          * read repair operation.
2194          */
2195         btrfs_bio_counter_inc_blocked(fs_info);
2196         if (btrfs_is_parity_mirror(fs_info, logical, length)) {
2197                 /*
2198                  * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2199                  * to update all raid stripes, but here we just want to correct
2200                  * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2201                  * stripe's dev and sector.
2202                  */
2203                 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2204                                       &map_length, &bbio, 0);
2205                 if (ret) {
2206                         btrfs_bio_counter_dec(fs_info);
2207                         bio_put(bio);
2208                         return -EIO;
2209                 }
2210                 ASSERT(bbio->mirror_num == 1);
2211         } else {
2212                 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2213                                       &map_length, &bbio, mirror_num);
2214                 if (ret) {
2215                         btrfs_bio_counter_dec(fs_info);
2216                         bio_put(bio);
2217                         return -EIO;
2218                 }
2219                 BUG_ON(mirror_num != bbio->mirror_num);
2220         }
2221
2222         sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2223         bio->bi_iter.bi_sector = sector;
2224         dev = bbio->stripes[bbio->mirror_num - 1].dev;
2225         btrfs_put_bbio(bbio);
2226         if (!dev || !dev->bdev ||
2227             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2228                 btrfs_bio_counter_dec(fs_info);
2229                 bio_put(bio);
2230                 return -EIO;
2231         }
2232         bio_set_dev(bio, dev->bdev);
2233         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2234         bio_add_page(bio, page, length, pg_offset);
2235
2236         if (btrfsic_submit_bio_wait(bio)) {
2237                 /* try to remap that extent elsewhere? */
2238                 btrfs_bio_counter_dec(fs_info);
2239                 bio_put(bio);
2240                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2241                 return -EIO;
2242         }
2243
2244         btrfs_info_rl_in_rcu(fs_info,
2245                 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2246                                   ino, start,
2247                                   rcu_str_deref(dev->name), sector);
2248         btrfs_bio_counter_dec(fs_info);
2249         bio_put(bio);
2250         return 0;
2251 }
2252
2253 int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num)
2254 {
2255         struct btrfs_fs_info *fs_info = eb->fs_info;
2256         u64 start = eb->start;
2257         int i, num_pages = num_extent_pages(eb);
2258         int ret = 0;
2259
2260         if (sb_rdonly(fs_info->sb))
2261                 return -EROFS;
2262
2263         for (i = 0; i < num_pages; i++) {
2264                 struct page *p = eb->pages[i];
2265
2266                 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2267                                         start - page_offset(p), mirror_num);
2268                 if (ret)
2269                         break;
2270                 start += PAGE_SIZE;
2271         }
2272
2273         return ret;
2274 }
2275
2276 /*
2277  * each time an IO finishes, we do a fast check in the IO failure tree
2278  * to see if we need to process or clean up an io_failure_record
2279  */
2280 int clean_io_failure(struct btrfs_fs_info *fs_info,
2281                      struct extent_io_tree *failure_tree,
2282                      struct extent_io_tree *io_tree, u64 start,
2283                      struct page *page, u64 ino, unsigned int pg_offset)
2284 {
2285         u64 private;
2286         struct io_failure_record *failrec;
2287         struct extent_state *state;
2288         int num_copies;
2289         int ret;
2290
2291         private = 0;
2292         ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2293                                EXTENT_DIRTY, 0);
2294         if (!ret)
2295                 return 0;
2296
2297         ret = get_state_failrec(failure_tree, start, &failrec);
2298         if (ret)
2299                 return 0;
2300
2301         BUG_ON(!failrec->this_mirror);
2302
2303         if (failrec->in_validation) {
2304                 /* there was no real error, just free the record */
2305                 btrfs_debug(fs_info,
2306                         "clean_io_failure: freeing dummy error at %llu",
2307                         failrec->start);
2308                 goto out;
2309         }
2310         if (sb_rdonly(fs_info->sb))
2311                 goto out;
2312
2313         spin_lock(&io_tree->lock);
2314         state = find_first_extent_bit_state(io_tree,
2315                                             failrec->start,
2316                                             EXTENT_LOCKED);
2317         spin_unlock(&io_tree->lock);
2318
2319         if (state && state->start <= failrec->start &&
2320             state->end >= failrec->start + failrec->len - 1) {
2321                 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2322                                               failrec->len);
2323                 if (num_copies > 1)  {
2324                         repair_io_failure(fs_info, ino, start, failrec->len,
2325                                           failrec->logical, page, pg_offset,
2326                                           failrec->failed_mirror);
2327                 }
2328         }
2329
2330 out:
2331         free_io_failure(failure_tree, io_tree, failrec);
2332
2333         return 0;
2334 }
2335
2336 /*
2337  * Can be called when
2338  * - hold extent lock
2339  * - under ordered extent
2340  * - the inode is freeing
2341  */
2342 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2343 {
2344         struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2345         struct io_failure_record *failrec;
2346         struct extent_state *state, *next;
2347
2348         if (RB_EMPTY_ROOT(&failure_tree->state))
2349                 return;
2350
2351         spin_lock(&failure_tree->lock);
2352         state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2353         while (state) {
2354                 if (state->start > end)
2355                         break;
2356
2357                 ASSERT(state->end <= end);
2358
2359                 next = next_state(state);
2360
2361                 failrec = state->failrec;
2362                 free_extent_state(state);
2363                 kfree(failrec);
2364
2365                 state = next;
2366         }
2367         spin_unlock(&failure_tree->lock);
2368 }
2369
2370 int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2371                 struct io_failure_record **failrec_ret)
2372 {
2373         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2374         struct io_failure_record *failrec;
2375         struct extent_map *em;
2376         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2377         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2378         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2379         int ret;
2380         u64 logical;
2381
2382         ret = get_state_failrec(failure_tree, start, &failrec);
2383         if (ret) {
2384                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2385                 if (!failrec)
2386                         return -ENOMEM;
2387
2388                 failrec->start = start;
2389                 failrec->len = end - start + 1;
2390                 failrec->this_mirror = 0;
2391                 failrec->bio_flags = 0;
2392                 failrec->in_validation = 0;
2393
2394                 read_lock(&em_tree->lock);
2395                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2396                 if (!em) {
2397                         read_unlock(&em_tree->lock);
2398                         kfree(failrec);
2399                         return -EIO;
2400                 }
2401
2402                 if (em->start > start || em->start + em->len <= start) {
2403                         free_extent_map(em);
2404                         em = NULL;
2405                 }
2406                 read_unlock(&em_tree->lock);
2407                 if (!em) {
2408                         kfree(failrec);
2409                         return -EIO;
2410                 }
2411
2412                 logical = start - em->start;
2413                 logical = em->block_start + logical;
2414                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2415                         logical = em->block_start;
2416                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2417                         extent_set_compress_type(&failrec->bio_flags,
2418                                                  em->compress_type);
2419                 }
2420
2421                 btrfs_debug(fs_info,
2422                         "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2423                         logical, start, failrec->len);
2424
2425                 failrec->logical = logical;
2426                 free_extent_map(em);
2427
2428                 /* set the bits in the private failure tree */
2429                 ret = set_extent_bits(failure_tree, start, end,
2430                                         EXTENT_LOCKED | EXTENT_DIRTY);
2431                 if (ret >= 0)
2432                         ret = set_state_failrec(failure_tree, start, failrec);
2433                 /* set the bits in the inode's tree */
2434                 if (ret >= 0)
2435                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2436                 if (ret < 0) {
2437                         kfree(failrec);
2438                         return ret;
2439                 }
2440         } else {
2441                 btrfs_debug(fs_info,
2442                         "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2443                         failrec->logical, failrec->start, failrec->len,
2444                         failrec->in_validation);
2445                 /*
2446                  * when data can be on disk more than twice, add to failrec here
2447                  * (e.g. with a list for failed_mirror) to make
2448                  * clean_io_failure() clean all those errors at once.
2449                  */
2450         }
2451
2452         *failrec_ret = failrec;
2453
2454         return 0;
2455 }
2456
2457 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2458                            struct io_failure_record *failrec, int failed_mirror)
2459 {
2460         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2461         int num_copies;
2462
2463         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2464         if (num_copies == 1) {
2465                 /*
2466                  * we only have a single copy of the data, so don't bother with
2467                  * all the retry and error correction code that follows. no
2468                  * matter what the error is, it is very likely to persist.
2469                  */
2470                 btrfs_debug(fs_info,
2471                         "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2472                         num_copies, failrec->this_mirror, failed_mirror);
2473                 return false;
2474         }
2475
2476         /*
2477          * there are two premises:
2478          *      a) deliver good data to the caller
2479          *      b) correct the bad sectors on disk
2480          */
2481         if (failed_bio_pages > 1) {
2482                 /*
2483                  * to fulfill b), we need to know the exact failing sectors, as
2484                  * we don't want to rewrite any more than the failed ones. thus,
2485                  * we need separate read requests for the failed bio
2486                  *
2487                  * if the following BUG_ON triggers, our validation request got
2488                  * merged. we need separate requests for our algorithm to work.
2489                  */
2490                 BUG_ON(failrec->in_validation);
2491                 failrec->in_validation = 1;
2492                 failrec->this_mirror = failed_mirror;
2493         } else {
2494                 /*
2495                  * we're ready to fulfill a) and b) alongside. get a good copy
2496                  * of the failed sector and if we succeed, we have setup
2497                  * everything for repair_io_failure to do the rest for us.
2498                  */
2499                 if (failrec->in_validation) {
2500                         BUG_ON(failrec->this_mirror != failed_mirror);
2501                         failrec->in_validation = 0;
2502                         failrec->this_mirror = 0;
2503                 }
2504                 failrec->failed_mirror = failed_mirror;
2505                 failrec->this_mirror++;
2506                 if (failrec->this_mirror == failed_mirror)
2507                         failrec->this_mirror++;
2508         }
2509
2510         if (failrec->this_mirror > num_copies) {
2511                 btrfs_debug(fs_info,
2512                         "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2513                         num_copies, failrec->this_mirror, failed_mirror);
2514                 return false;
2515         }
2516
2517         return true;
2518 }
2519
2520
2521 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2522                                     struct io_failure_record *failrec,
2523                                     struct page *page, int pg_offset, int icsum,
2524                                     bio_end_io_t *endio_func, void *data)
2525 {
2526         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2527         struct bio *bio;
2528         struct btrfs_io_bio *btrfs_failed_bio;
2529         struct btrfs_io_bio *btrfs_bio;
2530
2531         bio = btrfs_io_bio_alloc(1);
2532         bio->bi_end_io = endio_func;
2533         bio->bi_iter.bi_sector = failrec->logical >> 9;
2534         bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
2535         bio->bi_iter.bi_size = 0;
2536         bio->bi_private = data;
2537
2538         btrfs_failed_bio = btrfs_io_bio(failed_bio);
2539         if (btrfs_failed_bio->csum) {
2540                 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2541
2542                 btrfs_bio = btrfs_io_bio(bio);
2543                 btrfs_bio->csum = btrfs_bio->csum_inline;
2544                 icsum *= csum_size;
2545                 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2546                        csum_size);
2547         }
2548
2549         bio_add_page(bio, page, failrec->len, pg_offset);
2550
2551         return bio;
2552 }
2553
2554 /*
2555  * This is a generic handler for readpage errors. If other copies exist, read
2556  * those and write back good data to the failed position. Does not investigate
2557  * in remapping the failed extent elsewhere, hoping the device will be smart
2558  * enough to do this as needed
2559  */
2560 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2561                               struct page *page, u64 start, u64 end,
2562                               int failed_mirror)
2563 {
2564         struct io_failure_record *failrec;
2565         struct inode *inode = page->mapping->host;
2566         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2567         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2568         struct bio *bio;
2569         int read_mode = 0;
2570         blk_status_t status;
2571         int ret;
2572         unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
2573
2574         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2575
2576         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2577         if (ret)
2578                 return ret;
2579
2580         if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2581                                     failed_mirror)) {
2582                 free_io_failure(failure_tree, tree, failrec);
2583                 return -EIO;
2584         }
2585
2586         if (failed_bio_pages > 1)
2587                 read_mode |= REQ_FAILFAST_DEV;
2588
2589         phy_offset >>= inode->i_sb->s_blocksize_bits;
2590         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2591                                       start - page_offset(page),
2592                                       (int)phy_offset, failed_bio->bi_end_io,
2593                                       NULL);
2594         bio->bi_opf = REQ_OP_READ | read_mode;
2595
2596         btrfs_debug(btrfs_sb(inode->i_sb),
2597                 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2598                 read_mode, failrec->this_mirror, failrec->in_validation);
2599
2600         status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2601                                          failrec->bio_flags);
2602         if (status) {
2603                 free_io_failure(failure_tree, tree, failrec);
2604                 bio_put(bio);
2605                 ret = blk_status_to_errno(status);
2606         }
2607
2608         return ret;
2609 }
2610
2611 /* lots and lots of room for performance fixes in the end_bio funcs */
2612
2613 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2614 {
2615         int uptodate = (err == 0);
2616         int ret = 0;
2617
2618         btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
2619
2620         if (!uptodate) {
2621                 ClearPageUptodate(page);
2622                 SetPageError(page);
2623                 ret = err < 0 ? err : -EIO;
2624                 mapping_set_error(page->mapping, ret);
2625         }
2626 }
2627
2628 /*
2629  * after a writepage IO is done, we need to:
2630  * clear the uptodate bits on error
2631  * clear the writeback bits in the extent tree for this IO
2632  * end_page_writeback if the page has no more pending IO
2633  *
2634  * Scheduling is not allowed, so the extent state tree is expected
2635  * to have one and only one object corresponding to this IO.
2636  */
2637 static void end_bio_extent_writepage(struct bio *bio)
2638 {
2639         int error = blk_status_to_errno(bio->bi_status);
2640         struct bio_vec *bvec;
2641         u64 start;
2642         u64 end;
2643         struct bvec_iter_all iter_all;
2644
2645         ASSERT(!bio_flagged(bio, BIO_CLONED));
2646         bio_for_each_segment_all(bvec, bio, iter_all) {
2647                 struct page *page = bvec->bv_page;
2648                 struct inode *inode = page->mapping->host;
2649                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2650
2651                 /* We always issue full-page reads, but if some block
2652                  * in a page fails to read, blk_update_request() will
2653                  * advance bv_offset and adjust bv_len to compensate.
2654                  * Print a warning for nonzero offsets, and an error
2655                  * if they don't add up to a full page.  */
2656                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2657                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2658                                 btrfs_err(fs_info,
2659                                    "partial page write in btrfs with offset %u and length %u",
2660                                         bvec->bv_offset, bvec->bv_len);
2661                         else
2662                                 btrfs_info(fs_info,
2663                                    "incomplete page write in btrfs with offset %u and length %u",
2664                                         bvec->bv_offset, bvec->bv_len);
2665                 }
2666
2667                 start = page_offset(page);
2668                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2669
2670                 end_extent_writepage(page, error, start, end);
2671                 end_page_writeback(page);
2672         }
2673
2674         bio_put(bio);
2675 }
2676
2677 static void
2678 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2679                               int uptodate)
2680 {
2681         struct extent_state *cached = NULL;
2682         u64 end = start + len - 1;
2683
2684         if (uptodate && tree->track_uptodate)
2685                 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2686         unlock_extent_cached_atomic(tree, start, end, &cached);
2687 }
2688
2689 /*
2690  * after a readpage IO is done, we need to:
2691  * clear the uptodate bits on error
2692  * set the uptodate bits if things worked
2693  * set the page up to date if all extents in the tree are uptodate
2694  * clear the lock bit in the extent tree
2695  * unlock the page if there are no other extents locked for it
2696  *
2697  * Scheduling is not allowed, so the extent state tree is expected
2698  * to have one and only one object corresponding to this IO.
2699  */
2700 static void end_bio_extent_readpage(struct bio *bio)
2701 {
2702         struct bio_vec *bvec;
2703         int uptodate = !bio->bi_status;
2704         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2705         struct extent_io_tree *tree, *failure_tree;
2706         u64 offset = 0;
2707         u64 start;
2708         u64 end;
2709         u64 len;
2710         u64 extent_start = 0;
2711         u64 extent_len = 0;
2712         int mirror;
2713         int ret;
2714         struct bvec_iter_all iter_all;
2715
2716         ASSERT(!bio_flagged(bio, BIO_CLONED));
2717         bio_for_each_segment_all(bvec, bio, iter_all) {
2718                 struct page *page = bvec->bv_page;
2719                 struct inode *inode = page->mapping->host;
2720                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2721                 bool data_inode = btrfs_ino(BTRFS_I(inode))
2722                         != BTRFS_BTREE_INODE_OBJECTID;
2723
2724                 btrfs_debug(fs_info,
2725                         "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2726                         (u64)bio->bi_iter.bi_sector, bio->bi_status,
2727                         io_bio->mirror_num);
2728                 tree = &BTRFS_I(inode)->io_tree;
2729                 failure_tree = &BTRFS_I(inode)->io_failure_tree;
2730
2731                 /* We always issue full-page reads, but if some block
2732                  * in a page fails to read, blk_update_request() will
2733                  * advance bv_offset and adjust bv_len to compensate.
2734                  * Print a warning for nonzero offsets, and an error
2735                  * if they don't add up to a full page.  */
2736                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2737                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2738                                 btrfs_err(fs_info,
2739                                         "partial page read in btrfs with offset %u and length %u",
2740                                         bvec->bv_offset, bvec->bv_len);
2741                         else
2742                                 btrfs_info(fs_info,
2743                                         "incomplete page read in btrfs with offset %u and length %u",
2744                                         bvec->bv_offset, bvec->bv_len);
2745                 }
2746
2747                 start = page_offset(page);
2748                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2749                 len = bvec->bv_len;
2750
2751                 mirror = io_bio->mirror_num;
2752                 if (likely(uptodate)) {
2753                         ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2754                                                               page, start, end,
2755                                                               mirror);
2756                         if (ret)
2757                                 uptodate = 0;
2758                         else
2759                                 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2760                                                  failure_tree, tree, start,
2761                                                  page,
2762                                                  btrfs_ino(BTRFS_I(inode)), 0);
2763                 }
2764
2765                 if (likely(uptodate))
2766                         goto readpage_ok;
2767
2768                 if (data_inode) {
2769
2770                         /*
2771                          * The generic bio_readpage_error handles errors the
2772                          * following way: If possible, new read requests are
2773                          * created and submitted and will end up in
2774                          * end_bio_extent_readpage as well (if we're lucky,
2775                          * not in the !uptodate case). In that case it returns
2776                          * 0 and we just go on with the next page in our bio.
2777                          * If it can't handle the error it will return -EIO and
2778                          * we remain responsible for that page.
2779                          */
2780                         ret = bio_readpage_error(bio, offset, page, start, end,
2781                                                  mirror);
2782                         if (ret == 0) {
2783                                 uptodate = !bio->bi_status;
2784                                 offset += len;
2785                                 continue;
2786                         }
2787                 } else {
2788                         struct extent_buffer *eb;
2789
2790                         eb = (struct extent_buffer *)page->private;
2791                         set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2792                         eb->read_mirror = mirror;
2793                         atomic_dec(&eb->io_pages);
2794                         if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2795                                                &eb->bflags))
2796                                 btree_readahead_hook(eb, -EIO);
2797                 }
2798 readpage_ok:
2799                 if (likely(uptodate)) {
2800                         loff_t i_size = i_size_read(inode);
2801                         pgoff_t end_index = i_size >> PAGE_SHIFT;
2802                         unsigned off;
2803
2804                         /* Zero out the end if this page straddles i_size */
2805                         off = offset_in_page(i_size);
2806                         if (page->index == end_index && off)
2807                                 zero_user_segment(page, off, PAGE_SIZE);
2808                         SetPageUptodate(page);
2809                 } else {
2810                         ClearPageUptodate(page);
2811                         SetPageError(page);
2812                 }
2813                 unlock_page(page);
2814                 offset += len;
2815
2816                 if (unlikely(!uptodate)) {
2817                         if (extent_len) {
2818                                 endio_readpage_release_extent(tree,
2819                                                               extent_start,
2820                                                               extent_len, 1);
2821                                 extent_start = 0;
2822                                 extent_len = 0;
2823                         }
2824                         endio_readpage_release_extent(tree, start,
2825                                                       end - start + 1, 0);
2826                 } else if (!extent_len) {
2827                         extent_start = start;
2828                         extent_len = end + 1 - start;
2829                 } else if (extent_start + extent_len == start) {
2830                         extent_len += end + 1 - start;
2831                 } else {
2832                         endio_readpage_release_extent(tree, extent_start,
2833                                                       extent_len, uptodate);
2834                         extent_start = start;
2835                         extent_len = end + 1 - start;
2836                 }
2837         }
2838
2839         if (extent_len)
2840                 endio_readpage_release_extent(tree, extent_start, extent_len,
2841                                               uptodate);
2842         btrfs_io_bio_free_csum(io_bio);
2843         bio_put(bio);
2844 }
2845
2846 /*
2847  * Initialize the members up to but not including 'bio'. Use after allocating a
2848  * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2849  * 'bio' because use of __GFP_ZERO is not supported.
2850  */
2851 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
2852 {
2853         memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2854 }
2855
2856 /*
2857  * The following helpers allocate a bio. As it's backed by a bioset, it'll
2858  * never fail.  We're returning a bio right now but you can call btrfs_io_bio
2859  * for the appropriate container_of magic
2860  */
2861 struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
2862 {
2863         struct bio *bio;
2864
2865         bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
2866         bio_set_dev(bio, bdev);
2867         bio->bi_iter.bi_sector = first_byte >> 9;
2868         btrfs_io_bio_init(btrfs_io_bio(bio));
2869         return bio;
2870 }
2871
2872 struct bio *btrfs_bio_clone(struct bio *bio)
2873 {
2874         struct btrfs_io_bio *btrfs_bio;
2875         struct bio *new;
2876
2877         /* Bio allocation backed by a bioset does not fail */
2878         new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
2879         btrfs_bio = btrfs_io_bio(new);
2880         btrfs_io_bio_init(btrfs_bio);
2881         btrfs_bio->iter = bio->bi_iter;
2882         return new;
2883 }
2884
2885 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
2886 {
2887         struct bio *bio;
2888
2889         /* Bio allocation backed by a bioset does not fail */
2890         bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
2891         btrfs_io_bio_init(btrfs_io_bio(bio));
2892         return bio;
2893 }
2894
2895 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
2896 {
2897         struct bio *bio;
2898         struct btrfs_io_bio *btrfs_bio;
2899
2900         /* this will never fail when it's backed by a bioset */
2901         bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
2902         ASSERT(bio);
2903
2904         btrfs_bio = btrfs_io_bio(bio);
2905         btrfs_io_bio_init(btrfs_bio);
2906
2907         bio_trim(bio, offset >> 9, size >> 9);
2908         btrfs_bio->iter = bio->bi_iter;
2909         return bio;
2910 }
2911
2912 /*
2913  * @opf:        bio REQ_OP_* and REQ_* flags as one value
2914  * @tree:       tree so we can call our merge_bio hook
2915  * @wbc:        optional writeback control for io accounting
2916  * @page:       page to add to the bio
2917  * @pg_offset:  offset of the new bio or to check whether we are adding
2918  *              a contiguous page to the previous one
2919  * @size:       portion of page that we want to write
2920  * @offset:     starting offset in the page
2921  * @bdev:       attach newly created bios to this bdev
2922  * @bio_ret:    must be valid pointer, newly allocated bio will be stored there
2923  * @end_io_func:     end_io callback for new bio
2924  * @mirror_num:      desired mirror to read/write
2925  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
2926  * @bio_flags:  flags of the current bio to see if we can merge them
2927  */
2928 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2929                               struct writeback_control *wbc,
2930                               struct page *page, u64 offset,
2931                               size_t size, unsigned long pg_offset,
2932                               struct block_device *bdev,
2933                               struct bio **bio_ret,
2934                               bio_end_io_t end_io_func,
2935                               int mirror_num,
2936                               unsigned long prev_bio_flags,
2937                               unsigned long bio_flags,
2938                               bool force_bio_submit)
2939 {
2940         int ret = 0;
2941         struct bio *bio;
2942         size_t page_size = min_t(size_t, size, PAGE_SIZE);
2943         sector_t sector = offset >> 9;
2944
2945         ASSERT(bio_ret);
2946
2947         if (*bio_ret) {
2948                 bool contig;
2949                 bool can_merge = true;
2950
2951                 bio = *bio_ret;
2952                 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2953                         contig = bio->bi_iter.bi_sector == sector;
2954                 else
2955                         contig = bio_end_sector(bio) == sector;
2956
2957                 ASSERT(tree->ops);
2958                 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
2959                         can_merge = false;
2960
2961                 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2962                     force_bio_submit ||
2963                     bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2964                         ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2965                         if (ret < 0) {
2966                                 *bio_ret = NULL;
2967                                 return ret;
2968                         }
2969                         bio = NULL;
2970                 } else {
2971                         if (wbc)
2972                                 wbc_account_io(wbc, page, page_size);
2973                         return 0;
2974                 }
2975         }
2976
2977         bio = btrfs_bio_alloc(bdev, offset);
2978         bio_add_page(bio, page, page_size, pg_offset);
2979         bio->bi_end_io = end_io_func;
2980         bio->bi_private = tree;
2981         bio->bi_write_hint = page->mapping->host->i_write_hint;
2982         bio->bi_opf = opf;
2983         if (wbc) {
2984                 wbc_init_bio(wbc, bio);
2985                 wbc_account_io(wbc, page, page_size);
2986         }
2987
2988         *bio_ret = bio;
2989
2990         return ret;
2991 }
2992
2993 static void attach_extent_buffer_page(struct extent_buffer *eb,
2994                                       struct page *page)
2995 {
2996         if (!PagePrivate(page)) {
2997                 SetPagePrivate(page);
2998                 get_page(page);
2999                 set_page_private(page, (unsigned long)eb);
3000         } else {
3001                 WARN_ON(page->private != (unsigned long)eb);
3002         }
3003 }
3004
3005 void set_page_extent_mapped(struct page *page)
3006 {
3007         if (!PagePrivate(page)) {
3008                 SetPagePrivate(page);
3009                 get_page(page);
3010                 set_page_private(page, EXTENT_PAGE_PRIVATE);
3011         }
3012 }
3013
3014 static struct extent_map *
3015 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
3016                  u64 start, u64 len, get_extent_t *get_extent,
3017                  struct extent_map **em_cached)
3018 {
3019         struct extent_map *em;
3020
3021         if (em_cached && *em_cached) {
3022                 em = *em_cached;
3023                 if (extent_map_in_tree(em) && start >= em->start &&
3024                     start < extent_map_end(em)) {
3025                         refcount_inc(&em->refs);
3026                         return em;
3027                 }
3028
3029                 free_extent_map(em);
3030                 *em_cached = NULL;
3031         }
3032
3033         em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
3034         if (em_cached && !IS_ERR_OR_NULL(em)) {
3035                 BUG_ON(*em_cached);
3036                 refcount_inc(&em->refs);
3037                 *em_cached = em;
3038         }
3039         return em;
3040 }
3041 /*
3042  * basic readpage implementation.  Locked extent state structs are inserted
3043  * into the tree that are removed when the IO is done (by the end_io
3044  * handlers)
3045  * XXX JDM: This needs looking at to ensure proper page locking
3046  * return 0 on success, otherwise return error
3047  */
3048 static int __do_readpage(struct extent_io_tree *tree,
3049                          struct page *page,
3050                          get_extent_t *get_extent,
3051                          struct extent_map **em_cached,
3052                          struct bio **bio, int mirror_num,
3053                          unsigned long *bio_flags, unsigned int read_flags,
3054                          u64 *prev_em_start)
3055 {
3056         struct inode *inode = page->mapping->host;
3057         u64 start = page_offset(page);
3058         const u64 end = start + PAGE_SIZE - 1;
3059         u64 cur = start;
3060         u64 extent_offset;
3061         u64 last_byte = i_size_read(inode);
3062         u64 block_start;
3063         u64 cur_end;
3064         struct extent_map *em;
3065         struct block_device *bdev;
3066         int ret = 0;
3067         int nr = 0;
3068         size_t pg_offset = 0;
3069         size_t iosize;
3070         size_t disk_io_size;
3071         size_t blocksize = inode->i_sb->s_blocksize;
3072         unsigned long this_bio_flag = 0;
3073
3074         set_page_extent_mapped(page);
3075
3076         if (!PageUptodate(page)) {
3077                 if (cleancache_get_page(page) == 0) {
3078                         BUG_ON(blocksize != PAGE_SIZE);
3079                         unlock_extent(tree, start, end);
3080                         goto out;
3081                 }
3082         }
3083
3084         if (page->index == last_byte >> PAGE_SHIFT) {
3085                 char *userpage;
3086                 size_t zero_offset = offset_in_page(last_byte);
3087
3088                 if (zero_offset) {
3089                         iosize = PAGE_SIZE - zero_offset;
3090                         userpage = kmap_atomic(page);
3091                         memset(userpage + zero_offset, 0, iosize);
3092                         flush_dcache_page(page);
3093                         kunmap_atomic(userpage);
3094                 }
3095         }
3096         while (cur <= end) {
3097                 bool force_bio_submit = false;
3098                 u64 offset;
3099
3100                 if (cur >= last_byte) {
3101                         char *userpage;
3102                         struct extent_state *cached = NULL;
3103
3104                         iosize = PAGE_SIZE - pg_offset;
3105                         userpage = kmap_atomic(page);
3106                         memset(userpage + pg_offset, 0, iosize);
3107                         flush_dcache_page(page);
3108                         kunmap_atomic(userpage);
3109                         set_extent_uptodate(tree, cur, cur + iosize - 1,
3110                                             &cached, GFP_NOFS);
3111                         unlock_extent_cached(tree, cur,
3112                                              cur + iosize - 1, &cached);
3113                         break;
3114                 }
3115                 em = __get_extent_map(inode, page, pg_offset, cur,
3116                                       end - cur + 1, get_extent, em_cached);
3117                 if (IS_ERR_OR_NULL(em)) {
3118                         SetPageError(page);
3119                         unlock_extent(tree, cur, end);
3120                         break;
3121                 }
3122                 extent_offset = cur - em->start;
3123                 BUG_ON(extent_map_end(em) <= cur);
3124                 BUG_ON(end < cur);
3125
3126                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
3127                         this_bio_flag |= EXTENT_BIO_COMPRESSED;
3128                         extent_set_compress_type(&this_bio_flag,
3129                                                  em->compress_type);
3130                 }
3131
3132                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3133                 cur_end = min(extent_map_end(em) - 1, end);
3134                 iosize = ALIGN(iosize, blocksize);