fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/buffer_head.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/kthread.h>
  13 #include <linux/slab.h>
  14 #include <linux/migrate.h>
  15 #include <linux/ratelimit.h>
  16 #include <linux/uuid.h>
  17 #include <linux/semaphore.h>
  18 #include <linux/error-injection.h>
  19 #include <linux/crc32c.h>
  20 #include <asm/unaligned.h>
  21 #include "ctree.h"
  22 #include "disk-io.h"
  23 #include "transaction.h"
  24 #include "btrfs_inode.h"
  25 #include "volumes.h"
  26 #include "print-tree.h"
  27 #include "locking.h"
  28 #include "tree-log.h"
  29 #include "free-space-cache.h"
  30 #include "free-space-tree.h"
  31 #include "inode-map.h"
  32 #include "check-integrity.h"
  33 #include "rcu-string.h"
  34 #include "dev-replace.h"
  35 #include "raid56.h"
  36 #include "sysfs.h"
  37 #include "qgroup.h"
  38 #include "compression.h"
  39 #include "tree-checker.h"
  40 #include "ref-verify.h"
  41
  42 #ifdef CONFIG_X86
  43 #include <asm/cpufeature.h>
  44 #endif
  45
  46 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  47                                  BTRFS_HEADER_FLAG_RELOC |\
  48                                  BTRFS_SUPER_FLAG_ERROR |\
  49                                  BTRFS_SUPER_FLAG_SEEDING |\
  50                                  BTRFS_SUPER_FLAG_METADUMP |\
  51                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  52
  53 static const struct extent_io_ops btree_extent_io_ops;
  54 static void end_workqueue_fn(struct btrfs_work *work);
  55 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  56 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  57                                       struct btrfs_fs_info *fs_info);
  58 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  59 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  60                                         struct extent_io_tree *dirty_pages,
  61                                         int mark);
  62 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  63                                        struct extent_io_tree *pinned_extents);
  64 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  65 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  66
  67 /*
  68  * btrfs_end_io_wq structs are used to do processing in task context when an IO
  69  * is complete.  This is used during reads to verify checksums, and it is used
  70  * by writes to insert metadata for new file extents after IO is complete.
  71  */
  72 struct btrfs_end_io_wq {
  73         struct bio *bio;
  74         bio_end_io_t *end_io;
  75         void *private;
  76         struct btrfs_fs_info *info;
  77         blk_status_t status;
  78         enum btrfs_wq_endio_type metadata;
  79         struct btrfs_work work;
  80 };
  81
  82 static struct kmem_cache *btrfs_end_io_wq_cache;
  83
  84 int __init btrfs_end_io_wq_init(void)
  85 {
  86         btrfs_end_io_wq_cache = kmem_cache_create("btrfs_end_io_wq",
  87                                         sizeof(struct btrfs_end_io_wq),
  88                                         0,
  89                                         SLAB_MEM_SPREAD,
  90                                         NULL);
  91         if (!btrfs_end_io_wq_cache)
  92                 return -ENOMEM;
  93         return 0;
  94 }
  95
  96 void __cold btrfs_end_io_wq_exit(void)
  97 {
  98         kmem_cache_destroy(btrfs_end_io_wq_cache);
  99 }
 100
 101 /*
 102  * async submit bios are used to offload expensive checksumming
 103  * onto the worker threads.  They checksum file and metadata bios
 104  * just before they are sent down the IO stack.
 105  */
 106 struct async_submit_bio {
 107         void *private_data;
 108         struct bio *bio;
 109         extent_submit_bio_start_t *submit_bio_start;
 110         int mirror_num;
 111         /*
 112          * bio_offset is optional, can be used if the pages in the bio
 113          * can't tell us where in the file the bio should go
 114          */
 115         u64 bio_offset;
 116         struct btrfs_work work;
 117         blk_status_t status;
 118 };
 119
 120 /*
 121  * Lockdep class keys for extent_buffer->lock's in this root.  For a given
 122  * eb, the lockdep key is determined by the btrfs_root it belongs to and
 123  * the level the eb occupies in the tree.
 124  *
 125  * Different roots are used for different purposes and may nest inside each
 126  * other and they require separate keysets.  As lockdep keys should be
 127  * static, assign keysets according to the purpose of the root as indicated
 128  * by btrfs_root->root_key.objectid.  This ensures that all special purpose
 129  * roots have separate keysets.
 130  *
 131  * Lock-nesting across peer nodes is always done with the immediate parent
 132  * node locked thus preventing deadlock.  As lockdep doesn't know this, use
 133  * subclass to avoid triggering lockdep warning in such cases.
 134  *
 135  * The key is set by the readpage_end_io_hook after the buffer has passed
 136  * csum validation but before the pages are unlocked.  It is also set by
 137  * btrfs_init_new_buffer on freshly allocated blocks.
 138  *
 139  * We also add a check to make sure the highest level of the tree is the
 140  * same as our lockdep setup here.  If BTRFS_MAX_LEVEL changes, this code
 141  * needs update as well.
 142  */
 143 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 144 # if BTRFS_MAX_LEVEL != 8
 145 #  error
 146 # endif
 147
 148 static struct btrfs_lockdep_keyset {
 149         u64                     id;             /* root objectid */
 150         const char              *name_stem;     /* lock name stem */
 151         char                    names[BTRFS_MAX_LEVEL + 1][20];
 152         struct lock_class_key   keys[BTRFS_MAX_LEVEL + 1];
 153 } btrfs_lockdep_keysets[] = {
 154         { .id = BTRFS_ROOT_TREE_OBJECTID,       .name_stem = "root"     },
 155         { .id = BTRFS_EXTENT_TREE_OBJECTID,     .name_stem = "extent"   },
 156         { .id = BTRFS_CHUNK_TREE_OBJECTID,      .name_stem = "chunk"    },
 157         { .id = BTRFS_DEV_TREE_OBJECTID,        .name_stem = "dev"      },
 158         { .id = BTRFS_FS_TREE_OBJECTID,         .name_stem = "fs"       },
 159         { .id = BTRFS_CSUM_TREE_OBJECTID,       .name_stem = "csum"     },
 160         { .id = BTRFS_QUOTA_TREE_OBJECTID,      .name_stem = "quota"    },
 161         { .id = BTRFS_TREE_LOG_OBJECTID,        .name_stem = "log"      },
 162         { .id = BTRFS_TREE_RELOC_OBJECTID,      .name_stem = "treloc"   },
 163         { .id = BTRFS_DATA_RELOC_TREE_OBJECTID, .name_stem = "dreloc"   },
 164         { .id = BTRFS_UUID_TREE_OBJECTID,       .name_stem = "uuid"     },
 165         { .id = BTRFS_FREE_SPACE_TREE_OBJECTID, .name_stem = "free-space" },
 166         { .id = 0,                              .name_stem = "tree"     },
 167 };
 168
 169 void __init btrfs_init_lockdep(void)
 170 {
 171         int i, j;
 172
 173         /* initialize lockdep class names */
 174         for (i = 0; i < ARRAY_SIZE(btrfs_lockdep_keysets); i++) {
 175                 struct btrfs_lockdep_keyset *ks = &btrfs_lockdep_keysets[i];
 176
 177                 for (j = 0; j < ARRAY_SIZE(ks->names); j++)
 178                         snprintf(ks->names[j], sizeof(ks->names[j]),
 179                                  "btrfs-%s-%02d", ks->name_stem, j);
 180         }
 181 }
 182
 183 void btrfs_set_buffer_lockdep_class(u64 objectid, struct extent_buffer *eb,
 184                                     int level)
 185 {
 186         struct btrfs_lockdep_keyset *ks;
 187
 188         BUG_ON(level >= ARRAY_SIZE(ks->keys));
 189
 190         /* find the matching keyset, id 0 is the default entry */
 191         for (ks = btrfs_lockdep_keysets; ks->id; ks++)
 192                 if (ks->id == objectid)
 193                         break;
 194
 195         lockdep_set_class_and_name(&eb->lock,
 196                                    &ks->keys[level], ks->names[level]);
 197 }
 198
 199 #endif
 200
 201 /*
 202  * extents on the btree inode are pretty simple, there's one extent
 203  * that covers the entire device
 204  */
 205 struct extent_map *btree_get_extent(struct btrfs_inode *inode,
 206                 struct page *page, size_t pg_offset, u64 start, u64 len,
 207                 int create)
 208 {
 209         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 210         struct extent_map_tree *em_tree = &inode->extent_tree;
 211         struct extent_map *em;
 212         int ret;
 213
 214         read_lock(&em_tree->lock);
 215         em = lookup_extent_mapping(em_tree, start, len);
 216         if (em) {
 217                 em->bdev = fs_info->fs_devices->latest_bdev;
 218                 read_unlock(&em_tree->lock);
 219                 goto out;
 220         }
 221         read_unlock(&em_tree->lock);
 222
 223         em = alloc_extent_map();
 224         if (!em) {
 225                 em = ERR_PTR(-ENOMEM);
 226                 goto out;
 227         }
 228         em->start = 0;
 229         em->len = (u64)-1;
 230         em->block_len = (u64)-1;
 231         em->block_start = 0;
 232         em->bdev = fs_info->fs_devices->latest_bdev;
 233
 234         write_lock(&em_tree->lock);
 235         ret = add_extent_mapping(em_tree, em, 0);
 236         if (ret == -EEXIST) {
 237                 free_extent_map(em);
 238                 em = lookup_extent_mapping(em_tree, start, len);
 239                 if (!em)
 240                         em = ERR_PTR(-EIO);
 241         } else if (ret) {
 242                 free_extent_map(em);
 243                 em = ERR_PTR(ret);
 244         }
 245         write_unlock(&em_tree->lock);
 246
 247 out:
 248         return em;
 249 }
 250
 251 u32 btrfs_csum_data(const char *data, u32 seed, size_t len)
 252 {
 253         return crc32c(seed, data, len);
 254 }
 255
 256 void btrfs_csum_final(u32 crc, u8 *result)
 257 {
 258         put_unaligned_le32(~crc, result);
 259 }
 260
 261 /*
 262  * compute the csum for a btree block, and either verify it or write it
 263  * into the csum field of the block.
 264  */
 265 static int csum_tree_block(struct btrfs_fs_info *fs_info,
 266                            struct extent_buffer *buf,
 267                            int verify)
 268 {
 269         u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
 270         char result[BTRFS_CSUM_SIZE];
 271         unsigned long len;
 272         unsigned long cur_len;
 273         unsigned long offset = BTRFS_CSUM_SIZE;
 274         char *kaddr;
 275         unsigned long map_start;
 276         unsigned long map_len;
 277         int err;
 278         u32 crc = ~(u32)0;
 279
 280         len = buf->len - offset;
 281         while (len > 0) {
 282                 err = map_private_extent_buffer(buf, offset, 32,
 283                                         &kaddr, &map_start, &map_len);
 284                 if (err)
 285                         return err;
 286                 cur_len = min(len, map_len - (offset - map_start));
 287                 crc = btrfs_csum_data(kaddr + offset - map_start,
 288                                       crc, cur_len);
 289                 len -= cur_len;
 290                 offset += cur_len;
 291         }
 292         memset(result, 0, BTRFS_CSUM_SIZE);
 293
 294         btrfs_csum_final(crc, result);
 295
 296         if (verify) {
 297                 if (memcmp_extent_buffer(buf, result, 0, csum_size)) {
 298                         u32 val;
 299                         u32 found = 0;
 300                         memcpy(&found, result, csum_size);
 301
 302                         read_extent_buffer(buf, &val, 0, csum_size);
 303                         btrfs_warn_rl(fs_info,
 304                                 "%s checksum verify failed on %llu wanted %X found %X level %d",
 305                                 fs_info->sb->s_id, buf->start,
 306                                 val, found, btrfs_header_level(buf));
 307                         return -EUCLEAN;
 308                 }
 309         } else {
 310                 write_extent_buffer(buf, result, 0, csum_size);
 311         }
 312
 313         return 0;
 314 }
 315
 316 /*
 317  * we can't consider a given block up to date unless the transid of the
 318  * block matches the transid in the parent node's pointer.  This is how we
 319  * detect blocks that either didn't get written at all or got written
 320  * in the wrong place.
 321  */
 322 static int verify_parent_transid(struct extent_io_tree *io_tree,
 323                                  struct extent_buffer *eb, u64 parent_transid,
 324                                  int atomic)
 325 {
 326         struct extent_state *cached_state = NULL;
 327         int ret;
 328         bool need_lock = (current->journal_info == BTRFS_SEND_TRANS_STUB);
 329
 330         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 331                 return 0;
 332
 333         if (atomic)
 334                 return -EAGAIN;
 335
 336         if (need_lock) {
 337                 btrfs_tree_read_lock(eb);
 338                 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
 339         }
 340
 341         lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1,
 342                          &cached_state);
 343         if (extent_buffer_uptodate(eb) &&
 344             btrfs_header_generation(eb) == parent_transid) {
 345                 ret = 0;
 346                 goto out;
 347         }
 348         btrfs_err_rl(eb->fs_info,
 349                 "parent transid verify failed on %llu wanted %llu found %llu",
 350                         eb->start,
 351                         parent_transid, btrfs_header_generation(eb));
 352         ret = 1;
 353
 354         /*
 355          * Things reading via commit roots that don't have normal protection,
 356          * like send, can have a really old block in cache that may point at a
 357          * block that has been freed and re-allocated.  So don't clear uptodate
 358          * if we find an eb that is under IO (dirty/writeback) because we could
 359          * end up reading in the stale data and then writing it back out and
 360          * making everybody very sad.
 361          */
 362         if (!extent_buffer_under_io(eb))
 363                 clear_extent_buffer_uptodate(eb);
 364 out:
 365         unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1,
 366                              &cached_state);
 367         if (need_lock)
 368                 btrfs_tree_read_unlock_blocking(eb);
 369         return ret;
 370 }
 371
 372 /*
 373  * Return 0 if the superblock checksum type matches the checksum value of that
 374  * algorithm. Pass the raw disk superblock data.
 375  */
 376 static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 377                                   char *raw_disk_sb)
 378 {
 379         struct btrfs_super_block *disk_sb =
 380                 (struct btrfs_super_block *)raw_disk_sb;
 381         u16 csum_type = btrfs_super_csum_type(disk_sb);
 382         int ret = 0;
 383
 384         if (csum_type == BTRFS_CSUM_TYPE_CRC32) {
 385                 u32 crc = ~(u32)0;
 386                 char result[sizeof(crc)];
 387
 388                 /*
 389                  * The super_block structure does not span the whole
 390                  * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space
 391                  * is filled with zeros and is included in the checksum.
 392                  */
 393                 crc = btrfs_csum_data(raw_disk_sb + BTRFS_CSUM_SIZE,
 394                                 crc, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
 395                 btrfs_csum_final(crc, result);
 396
 397                 if (memcmp(raw_disk_sb, result, sizeof(result)))
 398                         ret = 1;
 399         }
 400
 401         if (csum_type >= ARRAY_SIZE(btrfs_csum_sizes)) {
 402                 btrfs_err(fs_info, "unsupported checksum algorithm %u",
 403                                 csum_type);
 404                 ret = 1;
 405         }
 406
 407         return ret;
 408 }
 409
 410 static int verify_level_key(struct btrfs_fs_info *fs_info,
 411                             struct extent_buffer *eb, int level,
 412                             struct btrfs_key *first_key, u64 parent_transid)
 413 {
 414         int found_level;
 415         struct btrfs_key found_key;
 416         int ret;
 417
 418         found_level = btrfs_header_level(eb);
 419         if (found_level != level) {
 420 #ifdef CONFIG_BTRFS_DEBUG
 421                 WARN_ON(1);
 422                 btrfs_err(fs_info,
 423 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 424                           eb->start, level, found_level);
 425 #endif
 426                 return -EIO;
 427         }
 428
 429         if (!first_key)
 430                 return 0;
 431
 432         /*
 433          * For live tree block (new tree blocks in current transaction),
 434          * we need proper lock context to avoid race, which is impossible here.
 435          * So we only checks tree blocks which is read from disk, whose
 436          * generation <= fs_info->last_trans_committed.
 437          */
 438         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 439                 return 0;
 440         if (found_level)
 441                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 442         else
 443                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 444         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 445
 446 #ifdef CONFIG_BTRFS_DEBUG
 447         if (ret) {
 448                 WARN_ON(1);
 449                 btrfs_err(fs_info,
 450 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 451                           eb->start, parent_transid, first_key->objectid,
 452                           first_key->type, first_key->offset,
 453                           found_key.objectid, found_key.type,
 454                           found_key.offset);
 455         }
 456 #endif
 457         return ret;
 458 }
 459
 460 /*
 461  * helper to read a given tree block, doing retries as required when
 462  * the checksums don't match and we have alternate mirrors to try.
 463  *
 464  * @parent_transid:     expected transid, skip check if 0
 465  * @level:              expected level, mandatory check
 466  * @first_key:          expected key of first slot, skip check if NULL
 467  */
 468 static int btree_read_extent_buffer_pages(struct btrfs_fs_info *fs_info,
 469                                           struct extent_buffer *eb,
 470                                           u64 parent_transid, int level,
 471                                           struct btrfs_key *first_key)
 472 {
 473         struct extent_io_tree *io_tree;
 474         int failed = 0;
 475         int ret;
 476         int num_copies = 0;
 477         int mirror_num = 0;
 478         int failed_mirror = 0;
 479
 480         clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 481         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
 482         while (1) {
 483                 ret = read_extent_buffer_pages(io_tree, eb, WAIT_COMPLETE,
 484                                                mirror_num);
 485                 if (!ret) {
 486                         if (verify_parent_transid(io_tree, eb,
 487                                                    parent_transid, 0))
 488                                 ret = -EIO;
 489                         else if (verify_level_key(fs_info, eb, level,
 490                                                   first_key, parent_transid))
 491                                 ret = -EUCLEAN;
 492                         else
 493                                 break;
 494                 }
 495
 496                 /*
 497                  * This buffer's crc is fine, but its contents are corrupted, so
 498                  * there is no reason to read the other copies, they won't be
 499                  * any less wrong.
 500                  */
 501                 if (test_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags) ||
 502                     ret == -EUCLEAN)
 503                         break;
 504
 505                 num_copies = btrfs_num_copies(fs_info,
 506                                               eb->start, eb->len);
 507                 if (num_copies == 1)
 508                         break;
 509
 510                 if (!failed_mirror) {
 511                         failed = 1;
 512                         failed_mirror = eb->read_mirror;
 513                 }
 514
 515                 mirror_num++;
 516                 if (mirror_num == failed_mirror)
 517                         mirror_num++;
 518
 519                 if (mirror_num > num_copies)
 520                         break;
 521         }
 522
 523         if (failed && !ret && failed_mirror)
 524                 repair_eb_io_failure(fs_info, eb, failed_mirror);
 525
 526         return ret;
 527 }
 528
 529 /*
 530  * checksum a dirty tree block before IO.  This has extra checks to make sure
 531  * we only fill in the checksum field in the first page of a multi-page block
 532  */
 533
 534 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct page *page)
 535 {
 536         u64 start = page_offset(page);
 537         u64 found_start;
 538         struct extent_buffer *eb;
 539
 540         eb = (struct extent_buffer *)page->private;
 541         if (page != eb->pages[0])
 542                 return 0;
 543
 544         found_start = btrfs_header_bytenr(eb);
 545         /*
 546          * Please do not consolidate these warnings into a single if.
 547          * It is useful to know what went wrong.
 548          */
 549         if (WARN_ON(found_start != start))
 550                 return -EUCLEAN;
 551         if (WARN_ON(!PageUptodate(page)))
 552                 return -EUCLEAN;
 553
 554         ASSERT(memcmp_extent_buffer(eb, fs_info->fsid,
 555                         btrfs_header_fsid(), BTRFS_FSID_SIZE) == 0);
 556
 557         return csum_tree_block(fs_info, eb, 0);
 558 }
 559
 560 static int check_tree_block_fsid(struct btrfs_fs_info *fs_info,
 561                                  struct extent_buffer *eb)
 562 {
 563         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 564         u8 fsid[BTRFS_FSID_SIZE];
 565         int ret = 1;
 566
 567         read_extent_buffer(eb, fsid, btrfs_header_fsid(), BTRFS_FSID_SIZE);
 568         while (fs_devices) {
 569                 if (!memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE)) {
 570                         ret = 0;
 571                         break;
 572                 }
 573                 fs_devices = fs_devices->seed;
 574         }
 575         return ret;
 576 }
 577
 578 static int btree_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
 579                                       u64 phy_offset, struct page *page,
 580                                       u64 start, u64 end, int mirror)
 581 {
 582         u64 found_start;
 583         int found_level;
 584         struct extent_buffer *eb;
 585         struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
 586         struct btrfs_fs_info *fs_info = root->fs_info;
 587         int ret = 0;
 588         int reads_done;
 589
 590         if (!page->private)
 591                 goto out;
 592
 593         eb = (struct extent_buffer *)page->private;
 594
 595         /* the pending IO might have been the only thing that kept this buffer
 596          * in memory.  Make sure we have a ref for all this other checks
 597          */
 598         extent_buffer_get(eb);
 599
 600         reads_done = atomic_dec_and_test(&eb->io_pages);
 601         if (!reads_done)
 602                 goto err;
 603
 604         eb->read_mirror = mirror;
 605         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 606                 ret = -EIO;
 607                 goto err;
 608         }
 609
 610         found_start = btrfs_header_bytenr(eb);
 611         if (found_start != eb->start) {
 612                 btrfs_err_rl(fs_info, "bad tree block start, want %llu have %llu",
 613                              eb->start, found_start);
 614                 ret = -EIO;
 615                 goto err;
 616         }
 617         if (check_tree_block_fsid(fs_info, eb)) {
 618                 btrfs_err_rl(fs_info, "bad fsid on block %llu",
 619                              eb->start);
 620                 ret = -EIO;
 621                 goto err;
 622         }
 623         found_level = btrfs_header_level(eb);
 624         if (found_level >= BTRFS_MAX_LEVEL) {
 625                 btrfs_err(fs_info, "bad tree block level %d on %llu",
 626                           (int)btrfs_header_level(eb), eb->start);
 627                 ret = -EIO;
 628                 goto err;
 629         }
 630
 631         btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb),
 632                                        eb, found_level);
 633
 634         ret = csum_tree_block(fs_info, eb, 1);
 635         if (ret)
 636                 goto err;
 637
 638         /*
 639          * If this is a leaf block and it is corrupt, set the corrupt bit so
 640          * that we don't try and read the other copies of this block, just
 641          * return -EIO.
 642          */
 643         if (found_level == 0 && btrfs_check_leaf_full(fs_info, eb)) {
 644                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 645                 ret = -EIO;
 646         }
 647
 648         if (found_level > 0 && btrfs_check_node(fs_info, eb))
 649                 ret = -EIO;
 650
 651         if (!ret)
 652                 set_extent_buffer_uptodate(eb);
 653 err:
 654         if (reads_done &&
 655             test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags))
 656                 btree_readahead_hook(eb, ret);
 657
 658         if (ret) {
 659                 /*
 660                  * our io error hook is going to dec the io pages
 661                  * again, we have to make sure it has something
 662                  * to decrement
 663                  */
 664                 atomic_inc(&eb->io_pages);
 665                 clear_extent_buffer_uptodate(eb);
 666         }
 667         free_extent_buffer(eb);
 668 out:
 669         return ret;
 670 }
 671
 672 static int btree_io_failed_hook(struct page *page, int failed_mirror)
 673 {
 674         struct extent_buffer *eb;
 675
 676         eb = (struct extent_buffer *)page->private;
 677         set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
 678         eb->read_mirror = failed_mirror;
 679         atomic_dec(&eb->io_pages);
 680         if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags))
 681                 btree_readahead_hook(eb, -EIO);
 682         return -EIO;    /* we fixed nothing */
 683 }
 684
 685 static void end_workqueue_bio(struct bio *bio)
 686 {
 687         struct btrfs_end_io_wq *end_io_wq = bio->bi_private;
 688         struct btrfs_fs_info *fs_info;
 689         struct btrfs_workqueue *wq;
 690         btrfs_work_func_t func;
 691
 692         fs_info = end_io_wq->info;
 693         end_io_wq->status = bio->bi_status;
 694
 695         if (bio_op(bio) == REQ_OP_WRITE) {
 696                 if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) {
 697                         wq = fs_info->endio_meta_write_workers;
 698                         func = btrfs_endio_meta_write_helper;
 699                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_FREE_SPACE) {
 700                         wq = fs_info->endio_freespace_worker;
 701                         func = btrfs_freespace_write_helper;
 702                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) {
 703                         wq = fs_info->endio_raid56_workers;
 704                         func = btrfs_endio_raid56_helper;
 705                 } else {
 706                         wq = fs_info->endio_write_workers;
 707                         func = btrfs_endio_write_helper;
 708                 }
 709         } else {
 710                 if (unlikely(end_io_wq->metadata ==
 711                              BTRFS_WQ_ENDIO_DIO_REPAIR)) {
 712                         wq = fs_info->endio_repair_workers;
 713                         func = btrfs_endio_repair_helper;
 714                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) {
 715                         wq = fs_info->endio_raid56_workers;
 716                         func = btrfs_endio_raid56_helper;
 717                 } else if (end_io_wq->metadata) {
 718                         wq = fs_info->endio_meta_workers;
 719                         func = btrfs_endio_meta_helper;
 720                 } else {
 721                         wq = fs_info->endio_workers;
 722                         func = btrfs_endio_helper;
 723                 }
 724         }
 725
 726         btrfs_init_work(&end_io_wq->work, func, end_workqueue_fn, NULL, NULL);
 727         btrfs_queue_work(wq, &end_io_wq->work);
 728 }
 729
 730 blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
 731                         enum btrfs_wq_endio_type metadata)
 732 {
 733         struct btrfs_end_io_wq *end_io_wq;
 734
 735         end_io_wq = kmem_cache_alloc(btrfs_end_io_wq_cache, GFP_NOFS);
 736         if (!end_io_wq)
 737                 return BLK_STS_RESOURCE;
 738
 739         end_io_wq->private = bio->bi_private;
 740         end_io_wq->end_io = bio->bi_end_io;
 741         end_io_wq->info = info;
 742         end_io_wq->status = 0;
 743         end_io_wq->bio = bio;
 744         end_io_wq->metadata = metadata;
 745
 746         bio->bi_private = end_io_wq;
 747         bio->bi_end_io = end_workqueue_bio;
 748         return 0;
 749 }
 750
 751 static void run_one_async_start(struct btrfs_work *work)
 752 {
 753         struct async_submit_bio *async;
 754         blk_status_t ret;
 755
 756         async = container_of(work, struct  async_submit_bio, work);
 757         ret = async->submit_bio_start(async->private_data, async->bio,
 758                                       async->bio_offset);
 759         if (ret)
 760                 async->status = ret;
 761 }
 762
 763 static void run_one_async_done(struct btrfs_work *work)
 764 {
 765         struct async_submit_bio *async;
 766
 767         async = container_of(work, struct  async_submit_bio, work);
 768
 769         /* If an error occurred we just want to clean up the bio and move on */
 770         if (async->status) {
 771                 async->bio->bi_status = async->status;
 772                 bio_endio(async->bio);
 773                 return;
 774         }
 775
 776         btrfs_submit_bio_done(async->private_data, async->bio, async->mirror_num);
 777 }
 778
 779 static void run_one_async_free(struct btrfs_work *work)
 780 {
 781         struct async_submit_bio *async;
 782
 783         async = container_of(work, struct  async_submit_bio, work);
 784         kfree(async);
 785 }
 786
 787 blk_status_t btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
 788                                  int mirror_num, unsigned long bio_flags,
 789                                  u64 bio_offset, void *private_data,
 790                                  extent_submit_bio_start_t *submit_bio_start)
 791 {
 792         struct async_submit_bio *async;
 793
 794         async = kmalloc(sizeof(*async), GFP_NOFS);
 795         if (!async)
 796                 return BLK_STS_RESOURCE;
 797
 798         async->private_data = private_data;
 799         async->bio = bio;
 800         async->mirror_num = mirror_num;
 801         async->submit_bio_start = submit_bio_start;
 802
 803         btrfs_init_work(&async->work, btrfs_worker_helper, run_one_async_start,
 804                         run_one_async_done, run_one_async_free);
 805
 806         async->bio_offset = bio_offset;
 807
 808         async->status = 0;
 809
 810         if (op_is_sync(bio->bi_opf))
 811                 btrfs_set_work_high_priority(&async->work);
 812
 813         btrfs_queue_work(fs_info->workers, &async->work);
 814         return 0;
 815 }
 816
 817 static blk_status_t btree_csum_one_bio(struct bio *bio)
 818 {
 819         struct bio_vec *bvec;
 820         struct btrfs_root *root;
 821         int i, ret = 0;
 822
 823         ASSERT(!bio_flagged(bio, BIO_CLONED));
 824         bio_for_each_segment_all(bvec, bio, i) {
 825                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
 826                 ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
 827                 if (ret)
 828                         break;
 829         }
 830
 831         return errno_to_blk_status(ret);
 832 }
 833
 834 static blk_status_t btree_submit_bio_start(void *private_data, struct bio *bio,
 835                                              u64 bio_offset)
 836 {
 837         /*
 838          * when we're called for a write, we're already in the async
 839          * submission context.  Just jump into btrfs_map_bio
 840          */
 841         return btree_csum_one_bio(bio);
 842 }
 843
 844 static int check_async_write(struct btrfs_inode *bi)
 845 {
 846         if (atomic_read(&bi->sync_writers))
 847                 return 0;
 848 #ifdef CONFIG_X86
 849         if (static_cpu_has(X86_FEATURE_XMM4_2))
 850                 return 0;
 851 #endif
 852         return 1;
 853 }
 854
 855 static blk_status_t btree_submit_bio_hook(void *private_data, struct bio *bio,
 856                                           int mirror_num, unsigned long bio_flags,
 857                                           u64 bio_offset)
 858 {
 859         struct inode *inode = private_data;
 860         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 861         int async = check_async_write(BTRFS_I(inode));
 862         blk_status_t ret;
 863
 864         if (bio_op(bio) != REQ_OP_WRITE) {
 865                 /*
 866                  * called for a read, do the setup so that checksum validation
 867                  * can happen in the async kernel threads
 868                  */
 869                 ret = btrfs_bio_wq_end_io(fs_info, bio,
 870                                           BTRFS_WQ_ENDIO_METADATA);
 871                 if (ret)
 872                         goto out_w_error;
 873                 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
 874         } else if (!async) {
 875                 ret = btree_csum_one_bio(bio);
 876                 if (ret)
 877                         goto out_w_error;
 878                 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
 879         } else {
 880                 /*
 881                  * kthread helpers are used to submit writes so that
 882                  * checksumming can happen in parallel across all CPUs
 883                  */
 884                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, 0,
 885                                           bio_offset, private_data,
 886                                           btree_submit_bio_start);
 887         }
 888
 889         if (ret)
 890                 goto out_w_error;
 891         return 0;
 892
 893 out_w_error:
 894         bio->bi_status = ret;
 895         bio_endio(bio);
 896         return ret;
 897 }
 898
 899 #ifdef CONFIG_MIGRATION
 900 static int btree_migratepage(struct address_space *mapping,
 901                         struct page *newpage, struct page *page,
 902                         enum migrate_mode mode)
 903 {
 904         /*
 905          * we can't safely write a btree page from here,
 906          * we haven't done the locking hook
 907          */
 908         if (PageDirty(page))
 909                 return -EAGAIN;
 910         /*
 911          * Buffers may be managed in a filesystem specific way.
 912          * We must have no buffers or drop them.
 913          */
 914         if (page_has_private(page) &&
 915             !try_to_release_page(page, GFP_KERNEL))
 916                 return -EAGAIN;
 917         return migrate_page(mapping, newpage, page, mode);
 918 }
 919 #endif
 920
 921
 922 static int btree_writepages(struct address_space *mapping,
 923                             struct writeback_control *wbc)
 924 {
 925         struct btrfs_fs_info *fs_info;
 926         int ret;
 927
 928         if (wbc->sync_mode == WB_SYNC_NONE) {
 929
 930                 if (wbc->for_kupdate)
 931                         return 0;
 932
 933                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 934                 /* this is a bit racy, but that's ok */
 935                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 936                                              BTRFS_DIRTY_METADATA_THRESH,
 937                                              fs_info->dirty_metadata_batch);
 938                 if (ret < 0)
 939                         return 0;
 940         }
 941         return btree_write_cache_pages(mapping, wbc);
 942 }
 943
 944 static int btree_readpage(struct file *file, struct page *page)
 945 {
 946         struct extent_io_tree *tree;
 947         tree = &BTRFS_I(page->mapping->host)->io_tree;
 948         return extent_read_full_page(tree, page, btree_get_extent, 0);
 949 }
 950
 951 static int btree_releasepage(struct page *page, gfp_t gfp_flags)
 952 {
 953         if (PageWriteback(page) || PageDirty(page))
 954                 return 0;
 955
 956         return try_release_extent_buffer(page);
 957 }
 958
 959 static void btree_invalidatepage(struct page *page, unsigned int offset,
 960                                  unsigned int length)
 961 {
 962         struct extent_io_tree *tree;
 963         tree = &BTRFS_I(page->mapping->host)->io_tree;
 964         extent_invalidatepage(tree, page, offset);
 965         btree_releasepage(page, GFP_NOFS);
 966         if (PagePrivate(page)) {
 967                 btrfs_warn(BTRFS_I(page->mapping->host)->root->fs_info,
 968                            "page private not zero on page %llu",
 969                            (unsigned long long)page_offset(page));
 970                 ClearPagePrivate(page);
 971                 set_page_private(page, 0);
 972                 put_page(page);
 973         }
 974 }
 975
 976 static int btree_set_page_dirty(struct page *page)
 977 {
 978 #ifdef DEBUG
 979         struct extent_buffer *eb;
 980
 981         BUG_ON(!PagePrivate(page));
 982         eb = (struct extent_buffer *)page->private;
 983         BUG_ON(!eb);
 984         BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 985         BUG_ON(!atomic_read(&eb->refs));
 986         btrfs_assert_tree_locked(eb);
 987 #endif
 988         return __set_page_dirty_nobuffers(page);
 989 }
 990
 991 static const struct address_space_operations btree_aops = {
 992         .readpage       = btree_readpage,
 993         .writepages     = btree_writepages,
 994         .releasepage    = btree_releasepage,
 995         .invalidatepage = btree_invalidatepage,
 996 #ifdef CONFIG_MIGRATION
 997         .migratepage    = btree_migratepage,
 998 #endif
 999         .set_page_dirty = btree_set_page_dirty,
1000 };
1001
1002 void readahead_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr)
1003 {
1004         struct extent_buffer *buf = NULL;
1005         struct inode *btree_inode = fs_info->btree_inode;
1006
1007         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1008         if (IS_ERR(buf))
1009                 return;
1010         read_extent_buffer_pages(&BTRFS_I(btree_inode)->io_tree,
1011                                  buf, WAIT_NONE, 0);
1012         free_extent_buffer(buf);
1013 }
1014
1015 int reada_tree_block_flagged(struct btrfs_fs_info *fs_info, u64 bytenr,
1016                          int mirror_num, struct extent_buffer **eb)
1017 {
1018         struct extent_buffer *buf = NULL;
1019         struct inode *btree_inode = fs_info->btree_inode;
1020         struct extent_io_tree *io_tree = &BTRFS_I(btree_inode)->io_tree;
1021         int ret;
1022
1023         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1024         if (IS_ERR(buf))
1025                 return 0;
1026
1027         set_bit(EXTENT_BUFFER_READAHEAD, &buf->bflags);
1028
1029         ret = read_extent_buffer_pages(io_tree, buf, WAIT_PAGE_LOCK,
1030                                        mirror_num);
1031         if (ret) {
1032                 free_extent_buffer(buf);
1033                 return ret;
1034         }
1035
1036         if (test_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags)) {
1037                 free_extent_buffer(buf);
1038                 return -EIO;
1039         } else if (extent_buffer_uptodate(buf)) {
1040                 *eb = buf;
1041         } else {
1042                 free_extent_buffer(buf);
1043         }
1044         return 0;
1045 }
1046
1047 struct extent_buffer *btrfs_find_create_tree_block(
1048                                                 struct btrfs_fs_info *fs_info,
1049                                                 u64 bytenr)
1050 {
1051         if (btrfs_is_testing(fs_info))
1052                 return alloc_test_extent_buffer(fs_info, bytenr);
1053         return alloc_extent_buffer(fs_info, bytenr);
1054 }
1055
1056
1057 int btrfs_write_tree_block(struct extent_buffer *buf)
1058 {
1059         return filemap_fdatawrite_range(buf->pages[0]->mapping, buf->start,
1060                                         buf->start + buf->len - 1);
1061 }
1062
1063 void btrfs_wait_tree_block_writeback(struct extent_buffer *buf)
1064 {
1065         filemap_fdatawait_range(buf->pages[0]->mapping,
1066                                 buf->start, buf->start + buf->len - 1);
1067 }
1068
1069 /*
1070  * Read tree block at logical address @bytenr and do variant basic but critical
1071  * verification.
1072  *
1073  * @parent_transid:     expected transid of this tree block, skip check if 0
1074  * @level:              expected level, mandatory check
1075  * @first_key:          expected key in slot 0, skip check if NULL
1076  */
1077 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
1078                                       u64 parent_transid, int level,
1079                                       struct btrfs_key *first_key)
1080 {
1081         struct extent_buffer *buf = NULL;
1082         int ret;
1083
1084         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1085         if (IS_ERR(buf))
1086                 return buf;
1087
1088         ret = btree_read_extent_buffer_pages(fs_info, buf, parent_transid,
1089                                              level, first_key);
1090         if (ret) {
1091                 free_extent_buffer(buf);
1092                 return ERR_PTR(ret);
1093         }
1094         return buf;
1095
1096 }
1097
1098 void clean_tree_block(struct btrfs_fs_info *fs_info,
1099                       struct extent_buffer *buf)
1100 {
1101         if (btrfs_header_generation(buf) ==
1102             fs_info->running_transaction->transid) {
1103                 btrfs_assert_tree_locked(buf);
1104
1105                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
1106                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1107                                                  -buf->len,
1108                                                  fs_info->dirty_metadata_batch);
1109                         /* ugh, clear_extent_buffer_dirty needs to lock the page */
1110                         btrfs_set_lock_blocking(buf);
1111                         clear_extent_buffer_dirty(buf);
1112                 }
1113         }
1114 }
1115
1116 static struct btrfs_subvolume_writers *btrfs_alloc_subvolume_writers(void)
1117 {
1118         struct btrfs_subvolume_writers *writers;
1119         int ret;
1120
1121         writers = kmalloc(sizeof(*writers), GFP_NOFS);
1122         if (!writers)
1123                 return ERR_PTR(-ENOMEM);
1124
1125         ret = percpu_counter_init(&writers->counter, 0, GFP_NOFS);
1126         if (ret < 0) {
1127                 kfree(writers);
1128                 return ERR_PTR(ret);
1129         }
1130
1131         init_waitqueue_head(&writers->wait);
1132         return writers;
1133 }
1134
1135 static void
1136 btrfs_free_subvolume_writers(struct btrfs_subvolume_writers *writers)
1137 {
1138         percpu_counter_destroy(&writers->counter);
1139         kfree(writers);
1140 }
1141
1142 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
1143                          u64 objectid)
1144 {
1145         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
1146         root->node = NULL;
1147         root->commit_root = NULL;
1148         root->state = 0;
1149         root->orphan_cleanup_state = 0;
1150
1151         root->last_trans = 0;
1152         root->highest_objectid = 0;
1153         root->nr_delalloc_inodes = 0;
1154         root->nr_ordered_extents = 0;
1155         root->inode_tree = RB_ROOT;
1156         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
1157         root->block_rsv = NULL;
1158
1159         INIT_LIST_HEAD(&root->dirty_list);
1160         INIT_LIST_HEAD(&root->root_list);
1161         INIT_LIST_HEAD(&root->delalloc_inodes);
1162         INIT_LIST_HEAD(&root->delalloc_root);
1163         INIT_LIST_HEAD(&root->ordered_extents);
1164         INIT_LIST_HEAD(&root->ordered_root);
1165         INIT_LIST_HEAD(&root->logged_list[0]);
1166         INIT_LIST_HEAD(&root->logged_list[1]);
1167         spin_lock_init(&root->inode_lock);
1168         spin_lock_init(&root->delalloc_lock);
1169         spin_lock_init(&root->ordered_extent_lock);
1170         spin_lock_init(&root->accounting_lock);
1171         spin_lock_init(&root->log_extents_lock[0]);
1172         spin_lock_init(&root->log_extents_lock[1]);
1173         spin_lock_init(&root->qgroup_meta_rsv_lock);
1174         mutex_init(&root->objectid_mutex);
1175         mutex_init(&root->log_mutex);
1176         mutex_init(&root->ordered_extent_mutex);
1177         mutex_init(&root->delalloc_mutex);
1178         init_waitqueue_head(&root->log_writer_wait);
1179         init_waitqueue_head(&root->log_commit_wait[0]);
1180         init_waitqueue_head(&root->log_commit_wait[1]);
1181         INIT_LIST_HEAD(&root->log_ctxs[0]);
1182         INIT_LIST_HEAD(&root->log_ctxs[1]);
1183         atomic_set(&root->log_commit[0], 0);
1184         atomic_set(&root->log_commit[1], 0);
1185         atomic_set(&root->log_writers, 0);
1186         atomic_set(&root->log_batch, 0);
1187         refcount_set(&root->refs, 1);
1188         atomic_set(&root->will_be_snapshotted, 0);
1189         atomic_set(&root->snapshot_force_cow, 0);
1190         root->log_transid = 0;
1191         root->log_transid_committed = -1;
1192         root->last_log_commit = 0;
1193         if (!dummy)
1194                 extent_io_tree_init(&root->dirty_log_pages, NULL);
1195
1196         memset(&root->root_key, 0, sizeof(root->root_key));
1197         memset(&root->root_item, 0, sizeof(root->root_item));
1198         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
1199         if (!dummy)
1200                 root->defrag_trans_start = fs_info->generation;
1201         else
1202                 root->defrag_trans_start = 0;
1203         root->root_key.objectid = objectid;
1204         root->anon_dev = 0;
1205
1206         spin_lock_init(&root->root_item_lock);
1207 }
1208
1209 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1210                 gfp_t flags)
1211 {
1212         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1213         if (root)
1214                 root->fs_info = fs_info;
1215         return root;
1216 }
1217
1218 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1219 /* Should only be used by the testing infrastructure */
1220 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1221 {
1222         struct btrfs_root *root;
1223
1224         if (!fs_info)
1225                 return ERR_PTR(-EINVAL);
1226
1227         root = btrfs_alloc_root(fs_info, GFP_KERNEL);
1228         if (!root)
1229                 return ERR_PTR(-ENOMEM);
1230
1231         /* We don't use the stripesize in selftest, set it as sectorsize */
1232         __setup_root(root, fs_info, BTRFS_ROOT_TREE_OBJECTID);
1233         root->alloc_bytenr = 0;
1234
1235         return root;
1236 }
1237 #endif
1238
1239 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1240                                      struct btrfs_fs_info *fs_info,
1241                                      u64 objectid)
1242 {
1243         struct extent_buffer *leaf;
1244         struct btrfs_root *tree_root = fs_info->tree_root;
1245         struct btrfs_root *root;
1246         struct btrfs_key key;
1247         int ret = 0;
1248         uuid_le uuid = NULL_UUID_LE;
1249
1250         root = btrfs_alloc_root(fs_info, GFP_KERNEL);
1251         if (!root)
1252                 return ERR_PTR(-ENOMEM);
1253
1254         __setup_root(root, fs_info, objectid);
1255         root->root_key.objectid = objectid;
1256         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1257         root->root_key.offset = 0;
1258
1259         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0);
1260         if (IS_ERR(leaf)) {
1261                 ret = PTR_ERR(leaf);
1262                 leaf = NULL;
1263                 goto fail;
1264         }
1265
1266         root->node = leaf;
1267         btrfs_mark_buffer_dirty(leaf);
1268
1269         root->commit_root = btrfs_root_node(root);
1270         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1271
1272         root->root_item.flags = 0;
1273         root->root_item.byte_limit = 0;
1274         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1275         btrfs_set_root_generation(&root->root_item, trans->transid);
1276         btrfs_set_root_level(&root->root_item, 0);
1277         btrfs_set_root_refs(&root->root_item, 1);
1278         btrfs_set_root_used(&root->root_item, leaf->len);
1279         btrfs_set_root_last_snapshot(&root->root_item, 0);
1280         btrfs_set_root_dirid(&root->root_item, 0);
1281         if (is_fstree(objectid))
1282                 uuid_le_gen(&uuid);
1283         memcpy(root->root_item.uuid, uuid.b, BTRFS_UUID_SIZE);
1284         root->root_item.drop_level = 0;
1285
1286         key.objectid = objectid;
1287         key.type = BTRFS_ROOT_ITEM_KEY;
1288         key.offset = 0;
1289         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1290         if (ret)
1291                 goto fail;
1292
1293         btrfs_tree_unlock(leaf);
1294
1295         return root;
1296
1297 fail:
1298         if (leaf) {
1299                 btrfs_tree_unlock(leaf);
1300                 free_extent_buffer(root->commit_root);
1301                 free_extent_buffer(leaf);
1302         }
1303         kfree(root);
1304
1305         return ERR_PTR(ret);
1306 }
1307
1308 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1309                                          struct btrfs_fs_info *fs_info)
1310 {
1311         struct btrfs_root *root;
1312         struct extent_buffer *leaf;
1313
1314         root = btrfs_alloc_root(fs_info, GFP_NOFS);
1315         if (!root)
1316                 return ERR_PTR(-ENOMEM);
1317
1318         __setup_root(root, fs_info, BTRFS_TREE_LOG_OBJECTID);
1319
1320         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1321         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1322         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1323
1324         /*
1325          * DON'T set REF_COWS for log trees
1326          *
1327          * log trees do not get reference counted because they go away
1328          * before a real commit is actually done.  They do store pointers
1329          * to file data extents, and those reference counts still get
1330          * updated (along with back refs to the log tree).
1331          */
1332
1333         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1334                         NULL, 0, 0, 0);
1335         if (IS_ERR(leaf)) {
1336                 kfree(root);
1337                 return ERR_CAST(leaf);
1338         }
1339
1340         root->node = leaf;
1341
1342         btrfs_mark_buffer_dirty(root->node);
1343         btrfs_tree_unlock(root->node);
1344         return root;
1345 }
1346
1347 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1348                              struct btrfs_fs_info *fs_info)
1349 {
1350         struct btrfs_root *log_root;
1351
1352         log_root = alloc_log_tree(trans, fs_info);
1353         if (IS_ERR(log_root))
1354                 return PTR_ERR(log_root);
1355         WARN_ON(fs_info->log_root_tree);
1356         fs_info->log_root_tree = log_root;
1357         return 0;
1358 }
1359
1360 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1361                        struct btrfs_root *root)
1362 {
1363         struct btrfs_fs_info *fs_info = root->fs_info;
1364         struct btrfs_root *log_root;
1365         struct btrfs_inode_item *inode_item;
1366
1367         log_root = alloc_log_tree(trans, fs_info);
1368         if (IS_ERR(log_root))
1369                 return PTR_ERR(log_root);
1370
1371         log_root->last_trans = trans->transid;
1372         log_root->root_key.offset = root->root_key.objectid;
1373
1374         inode_item = &log_root->root_item.inode;
1375         btrfs_set_stack_inode_generation(inode_item, 1);
1376         btrfs_set_stack_inode_size(inode_item, 3);
1377         btrfs_set_stack_inode_nlink(inode_item, 1);
1378         btrfs_set_stack_inode_nbytes(inode_item,
1379                                      fs_info->nodesize);
1380         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1381
1382         btrfs_set_root_node(&log_root->root_item, log_root->node);
1383
1384         WARN_ON(root->log_root);
1385         root->log_root = log_root;
1386         root->log_transid = 0;
1387         root->log_transid_committed = -1;
1388         root->last_log_commit = 0;
1389         return 0;
1390 }
1391
1392 static struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1393                                                struct btrfs_key *key)
1394 {
1395         struct btrfs_root *root;
1396         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1397         struct btrfs_path *path;
1398         u64 generation;
1399         int ret;
1400         int level;
1401
1402         path = btrfs_alloc_path();
1403         if (!path)
1404                 return ERR_PTR(-ENOMEM);
1405
1406         root = btrfs_alloc_root(fs_info, GFP_NOFS);
1407         if (!root) {
1408                 ret = -ENOMEM;
1409                 goto alloc_fail;
1410         }
1411
1412         __setup_root(root, fs_info, key->objectid);
1413
1414         ret = btrfs_find_root(tree_root, key, path,
1415                               &root->root_item, &root->root_key);
1416         if (ret) {
1417                 if (ret > 0)
1418                         ret = -ENOENT;
1419                 goto find_fail;
1420         }
1421
1422         generation = btrfs_root_generation(&root->root_item);
1423         level = btrfs_root_level(&root->root_item);
1424         root->node = read_tree_block(fs_info,
1425                                      btrfs_root_bytenr(&root->root_item),
1426                                      generation, level, NULL);
1427         if (IS_ERR(root->node)) {
1428                 ret = PTR_ERR(root->node);
1429                 goto find_fail;
1430         } else if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1431                 ret = -EIO;
1432                 free_extent_buffer(root->node);
1433                 goto find_fail;
1434         }
1435         root->commit_root = btrfs_root_node(root);
1436 out:
1437         btrfs_free_path(path);
1438         return root;
1439
1440 find_fail:
1441         kfree(root);
1442 alloc_fail:
1443         root = ERR_PTR(ret);
1444         goto out;
1445 }
1446
1447 struct btrfs_root *btrfs_read_fs_root(struct btrfs_root *tree_root,
1448                                       struct btrfs_key *location)
1449 {
1450         struct btrfs_root *root;
1451
1452         root = btrfs_read_tree_root(tree_root, location);
1453         if (IS_ERR(root))
1454                 return root;
1455
1456         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
1457                 set_bit(BTRFS_ROOT_REF_COWS, &root->state);
1458                 btrfs_check_and_init_root_item(&root->root_item);
1459         }
1460
1461         return root;
1462 }
1463
1464 int btrfs_init_fs_root(struct btrfs_root *root)
1465 {
1466         int ret;
1467         struct btrfs_subvolume_writers *writers;
1468
1469         root->free_ino_ctl = kzalloc(sizeof(*root->free_ino_ctl), GFP_NOFS);
1470         root->free_ino_pinned = kzalloc(sizeof(*root->free_ino_pinned),
1471                                         GFP_NOFS);
1472         if (!root->free_ino_pinned || !root->free_ino_ctl) {
1473                 ret = -ENOMEM;
1474                 goto fail;
1475         }
1476
1477         writers = btrfs_alloc_subvolume_writers();
1478         if (IS_ERR(writers)) {
1479                 ret = PTR_ERR(writers);
1480                 goto fail;
1481         }
1482         root->subv_writers = writers;
1483
1484         btrfs_init_free_ino_ctl(root);
1485         spin_lock_init(&root->ino_cache_lock);
1486         init_waitqueue_head(&root->ino_cache_wait);
1487
1488         ret = get_anon_bdev(&root->anon_dev);
1489         if (ret)
1490                 goto fail;
1491
1492         mutex_lock(&root->objectid_mutex);
1493         ret = btrfs_find_highest_objectid(root,
1494                                         &root->highest_objectid);
1495         if (ret) {
1496                 mutex_unlock(&root->objectid_mutex);
1497                 goto fail;
1498         }
1499
1500         ASSERT(root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
1501
1502         mutex_unlock(&root->objectid_mutex);
1503
1504         return 0;
1505 fail:
1506         /* The caller is responsible to call btrfs_free_fs_root */
1507         return ret;
1508 }
1509
1510 struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1511                                         u64 root_id)
1512 {
1513         struct btrfs_root *root;
1514
1515         spin_lock(&fs_info->fs_roots_radix_lock);
1516         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1517                                  (unsigned long)root_id);
1518         spin_unlock(&fs_info->fs_roots_radix_lock);
1519         return root;
1520 }
1521
1522 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1523                          struct btrfs_root *root)
1524 {
1525         int ret;
1526
1527         ret = radix_tree_preload(GFP_NOFS);
1528         if (ret)
1529                 return ret;
1530
1531         spin_lock(&fs_info->fs_roots_radix_lock);
1532         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1533                                 (unsigned long)root->root_key.objectid,
1534                                 root);
1535         if (ret == 0)
1536                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1537         spin_unlock(&fs_info->fs_roots_radix_lock);
1538         radix_tree_preload_end();
1539
1540         return ret;
1541 }
1542
1543 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1544                                      struct btrfs_key *location,
1545                                      bool check_ref)
1546 {
1547         struct btrfs_root *root;
1548         struct btrfs_path *path;
1549         struct btrfs_key key;
1550         int ret;
1551
1552         if (location->objectid == BTRFS_ROOT_TREE_OBJECTID)
1553                 return fs_info->tree_root;
1554         if (location->objectid == BTRFS_EXTENT_TREE_OBJECTID)
1555                 return fs_info->extent_root;
1556         if (location->objectid == BTRFS_CHUNK_TREE_OBJECTID)
1557                 return fs_info->chunk_root;
1558         if (location->objectid == BTRFS_DEV_TREE_OBJECTID)
1559                 return fs_info->dev_root;
1560         if (location->objectid == BTRFS_CSUM_TREE_OBJECTID)
1561                 return fs_info->csum_root;
1562         if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID)
1563                 return fs_info->quota_root ? fs_info->quota_root :
1564                                              ERR_PTR(-ENOENT);
1565         if (location->objectid == BTRFS_UUID_TREE_OBJECTID)
1566                 return fs_info->uuid_root ? fs_info->uuid_root :
1567                                             ERR_PTR(-ENOENT);
1568         if (location->objectid == BTRFS_FREE_SPACE_TREE_OBJECTID)
1569                 return fs_info->free_space_root ? fs_info->free_space_root :
1570                                                   ERR_PTR(-ENOENT);
1571 again:
1572         root = btrfs_lookup_fs_root(fs_info, location->objectid);
1573         if (root) {
1574                 if (check_ref && btrfs_root_refs(&root->root_item) == 0)
1575                         return ERR_PTR(-ENOENT);
1576                 return root;
1577         }
1578
1579         root = btrfs_read_fs_root(fs_info->tree_root, location);
1580         if (IS_ERR(root))
1581                 return root;
1582
1583         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1584                 ret = -ENOENT;
1585                 goto fail;
1586         }
1587
1588         ret = btrfs_init_fs_root(root);
1589         if (ret)
1590                 goto fail;
1591
1592         path = btrfs_alloc_path();
1593         if (!path) {
1594                 ret = -ENOMEM;
1595                 goto fail;
1596         }
1597         key.objectid = BTRFS_ORPHAN_OBJECTID;
1598         key.type = BTRFS_ORPHAN_ITEM_KEY;
1599         key.offset = location->objectid;
1600
1601         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1602         btrfs_free_path(path);
1603         if (ret < 0)
1604                 goto fail;
1605         if (ret == 0)
1606                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1607
1608         ret = btrfs_insert_fs_root(fs_info, root);
1609         if (ret) {
1610                 if (ret == -EEXIST) {
1611                         btrfs_free_fs_root(root);
1612                         goto again;
1613                 }
1614                 goto fail;
1615         }
1616         return root;
1617 fail:
1618         btrfs_free_fs_root(root);
1619         return ERR_PTR(ret);
1620 }
1621
1622 static int btrfs_congested_fn(void *congested_data, int bdi_bits)
1623 {
1624         struct btrfs_fs_info *info = (struct btrfs_fs_info *)congested_data;
1625         int ret = 0;
1626         struct btrfs_device *device;
1627         struct backing_dev_info *bdi;
1628
1629         rcu_read_lock();
1630         list_for_each_entry_rcu(device, &info->fs_devices->devices, dev_list) {
1631                 if (!device->bdev)
1632                         continue;
1633                 bdi = device->bdev->bd_bdi;
1634                 if (bdi_congested(bdi, bdi_bits)) {
1635                         ret = 1;
1636                         break;
1637                 }
1638         }
1639         rcu_read_unlock();
1640         return ret;
1641 }
1642
1643 /*
1644  * called by the kthread helper functions to finally call the bio end_io
1645  * functions.  This is where read checksum verification actually happens
1646  */
1647 static void end_workqueue_fn(struct btrfs_work *work)
1648 {
1649         struct bio *bio;
1650         struct btrfs_end_io_wq *end_io_wq;
1651
1652         end_io_wq = container_of(work, struct btrfs_end_io_wq, work);
1653         bio = end_io_wq->bio;
1654
1655         bio->bi_status = end_io_wq->status;
1656         bio->bi_private = end_io_wq->private;
1657         bio->bi_end_io = end_io_wq->end_io;
1658         kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
1659         bio_endio(bio);
1660 }
1661
1662 static int cleaner_kthread(void *arg)
1663 {
1664         struct btrfs_root *root = arg;
1665         struct btrfs_fs_info *fs_info = root->fs_info;
1666         int again;
1667         struct btrfs_trans_handle *trans;
1668
1669         do {
1670                 again = 0;
1671
1672                 /* Make the cleaner go to sleep early. */
1673                 if (btrfs_need_cleaner_sleep(fs_info))
1674                         goto sleep;
1675
1676                 /*
1677                  * Do not do anything if we might cause open_ctree() to block
1678                  * before we have finished mounting the filesystem.
1679                  */
1680                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1681                         goto sleep;
1682
1683                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1684                         goto sleep;
1685
1686                 /*
1687                  * Avoid the problem that we change the status of the fs
1688                  * during the above check and trylock.
1689                  */
1690                 if (btrfs_need_cleaner_sleep(fs_info)) {
1691                         mutex_unlock(&fs_info->cleaner_mutex);
1692                         goto sleep;
1693                 }
1694
1695                 mutex_lock(&fs_info->cleaner_delayed_iput_mutex);
1696                 btrfs_run_delayed_iputs(fs_info);
1697                 mutex_unlock(&fs_info->cleaner_delayed_iput_mutex);
1698
1699                 again = btrfs_clean_one_deleted_snapshot(root);
1700                 mutex_unlock(&fs_info->cleaner_mutex);
1701
1702                 /*
1703                  * The defragger has dealt with the R/O remount and umount,
1704                  * needn't do anything special here.
1705                  */
1706                 btrfs_run_defrag_inodes(fs_info);
1707
1708                 /*
1709                  * Acquires fs_info->delete_unused_bgs_mutex to avoid racing
1710                  * with relocation (btrfs_relocate_chunk) and relocation
1711                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1712                  * after acquiring fs_info->delete_unused_bgs_mutex. So we
1713                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1714                  * unused block groups.
1715                  */
1716                 btrfs_delete_unused_bgs(fs_info);
1717 sleep:
1718                 if (!again) {
1719                         set_current_state(TASK_INTERRUPTIBLE);
1720                         if (!kthread_should_stop())
1721                                 schedule();
1722                         __set_current_state(TASK_RUNNING);
1723                 }
1724         } while (!kthread_should_stop());
1725
1726         /*
1727          * Transaction kthread is stopped before us and wakes us up.
1728          * However we might have started a new transaction and COWed some
1729          * tree blocks when deleting unused block groups for example. So
1730          * make sure we commit the transaction we started to have a clean
1731          * shutdown when evicting the btree inode - if it has dirty pages
1732          * when we do the final iput() on it, eviction will trigger a
1733          * writeback for it which will fail with null pointer dereferences
1734          * since work queues and other resources were already released and
1735          * destroyed by the time the iput/eviction/writeback is made.
1736          */
1737         trans = btrfs_attach_transaction(root);
1738         if (IS_ERR(trans)) {
1739                 if (PTR_ERR(trans) != -ENOENT)
1740                         btrfs_err(fs_info,
1741                                   "cleaner transaction attach returned %ld",
1742                                   PTR_ERR(trans));
1743         } else {
1744                 int ret;
1745
1746                 ret = btrfs_commit_transaction(trans);
1747                 if (ret)
1748                         btrfs_err(fs_info,
1749                                   "cleaner open transaction commit returned %d",
1750                                   ret);
1751         }
1752
1753         return 0;
1754 }
1755
1756 static int transaction_kthread(void *arg)
1757 {
1758         struct btrfs_root *root = arg;
1759         struct btrfs_fs_info *fs_info = root->fs_info;
1760         struct btrfs_trans_handle *trans;
1761         struct btrfs_transaction *cur;
1762         u64 transid;
1763         time64_t now;
1764         unsigned long delay;
1765         bool cannot_commit;
1766
1767         do {
1768                 cannot_commit = false;
1769                 delay = HZ * fs_info->commit_interval;
1770                 mutex_lock(&fs_info->transaction_kthread_mutex);
1771
1772                 spin_lock(&fs_info->trans_lock);
1773                 cur = fs_info->running_transaction;
1774                 if (!cur) {
1775                         spin_unlock(&fs_info->trans_lock);
1776                         goto sleep;
1777                 }
1778
1779                 now = ktime_get_seconds();
1780                 if (cur->state < TRANS_STATE_BLOCKED &&
1781                     !test_bit(BTRFS_FS_NEED_ASYNC_COMMIT, &fs_info->flags) &&
1782                     (now < cur->start_time ||
1783                      now - cur->start_time < fs_info->commit_interval)) {
1784                         spin_unlock(&fs_info->trans_lock);
1785                         delay = HZ * 5;
1786                         goto sleep;
1787                 }
1788                 transid = cur->transid;
1789                 spin_unlock(&fs_info->trans_lock);
1790
1791                 /* If the file system is aborted, this will always fail. */
1792                 trans = btrfs_attach_transaction(root);
1793                 if (IS_ERR(trans)) {
1794                         if (PTR_ERR(trans) != -ENOENT)
1795                                 cannot_commit = true;
1796                         goto sleep;
1797                 }
1798                 if (transid == trans->transid) {
1799                         btrfs_commit_transaction(trans);
1800                 } else {
1801                         btrfs_end_transaction(trans);
1802                 }
1803 sleep:
1804                 wake_up_process(fs_info->cleaner_kthread);
1805                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1806
1807                 if (unlikely(test_bit(BTRFS_FS_STATE_ERROR,
1808                                       &fs_info->fs_state)))
1809                         btrfs_cleanup_transaction(fs_info);
1810                 if (!kthread_should_stop() &&
1811                                 (!btrfs_transaction_blocked(fs_info) ||
1812                                  cannot_commit))
1813                         schedule_timeout_interruptible(delay);
1814         } while (!kthread_should_stop());
1815         return 0;
1816 }
1817
1818 /*
1819  * this will find the highest generation in the array of
1820  * root backups.  The index of the highest array is returned,
1821  * or -1 if we can't find anything.
1822  *
1823  * We check to make sure the array is valid by comparing the
1824  * generation of the latest  root in the array with the generation
1825  * in the super block.  If they don't match we pitch it.
1826  */
1827 static int find_newest_super_backup(struct btrfs_fs_info *info, u64 newest_gen)
1828 {
1829         u64 cur;
1830         int newest_index = -1;
1831         struct btrfs_root_backup *root_backup;
1832         int i;
1833
1834         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1835                 root_backup = info->super_copy->super_roots + i;
1836                 cur = btrfs_backup_tree_root_gen(root_backup);
1837                 if (cur == newest_gen)
1838                         newest_index = i;
1839         }
1840
1841         /* check to see if we actually wrapped around */
1842         if (newest_index == BTRFS_NUM_BACKUP_ROOTS - 1) {
1843                 root_backup = info->super_copy->super_roots;
1844                 cur = btrfs_backup_tree_root_gen(root_backup);
1845                 if (cur == newest_gen)
1846                         newest_index = 0;
1847         }
1848         return newest_index;
1849 }
1850
1851
1852 /*
1853  * find the oldest backup so we know where to store new entries
1854  * in the backup array.  This will set the backup_root_index
1855  * field in the fs_info struct
1856  */
1857 static void find_oldest_super_backup(struct btrfs_fs_info *info,
1858                                      u64 newest_gen)
1859 {
1860         int newest_index = -1;
1861
1862         newest_index = find_newest_super_backup(info, newest_gen);
1863         /* if there was garbage in there, just move along */
1864         if (newest_index == -1) {
1865                 info->backup_root_index = 0;
1866         } else {
1867                 info->backup_root_index = (newest_index + 1) % BTRFS_NUM_BACKUP_ROOTS;
1868         }
1869 }
1870
1871 /*
1872  * copy all the root pointers into the super backup array.
1873  * this will bump the backup pointer by one when it is
1874  * done
1875  */
1876 static void backup_super_roots(struct btrfs_fs_info *info)
1877 {
1878         int next_backup;
1879         struct btrfs_root_backup *root_backup;
1880         int last_backup;
1881
1882         next_backup = info->backup_root_index;
1883         last_backup = (next_backup + BTRFS_NUM_BACKUP_ROOTS - 1) %
1884                 BTRFS_NUM_BACKUP_ROOTS;
1885
1886         /*
1887          * just overwrite the last backup if we're at the same generation
1888          * this happens only at umount
1889          */
1890         root_backup = info->super_for_commit->super_roots + last_backup;
1891         if (btrfs_backup_tree_root_gen(root_backup) ==
1892             btrfs_header_generation(info->tree_root->node))
1893                 next_backup = last_backup;
1894
1895         root_backup = info->super_for_commit->super_roots + next_backup;
1896
1897         /*
1898          * make sure all of our padding and empty slots get zero filled
1899          * regardless of which ones we use today
1900          */
1901         memset(root_backup, 0, sizeof(*root_backup));
1902
1903         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1904
1905         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1906         btrfs_set_backup_tree_root_gen(root_backup,
1907                                btrfs_header_generation(info->tree_root->node));
1908
1909         btrfs_set_backup_tree_root_level(root_backup,
1910                                btrfs_header_level(info->tree_root->node));
1911
1912         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1913         btrfs_set_backup_chunk_root_gen(root_backup,
1914                                btrfs_header_generation(info->chunk_root->node));
1915         btrfs_set_backup_chunk_root_level(root_backup,
1916                                btrfs_header_level(info->chunk_root->node));
1917
1918         btrfs_set_backup_extent_root(root_backup, info->extent_root->node->start);
1919         btrfs_set_backup_extent_root_gen(root_backup,
1920                                btrfs_header_generation(info->extent_root->node));
1921         btrfs_set_backup_extent_root_level(root_backup,
1922                                btrfs_header_level(info->extent_root->node));
1923
1924         /*
1925          * we might commit during log recovery, which happens before we set
1926          * the fs_root.  Make sure it is valid before we fill it in.
1927          */
1928         if (info->fs_root && info->fs_root->node) {
1929                 btrfs_set_backup_fs_root(root_backup,
1930                                          info->fs_root->node->start);
1931                 btrfs_set_backup_fs_root_gen(root_backup,
1932                                btrfs_header_generation(info->fs_root->node));
1933                 btrfs_set_backup_fs_root_level(root_backup,
1934                                btrfs_header_level(info->fs_root->node));
1935         }
1936
1937         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
1938         btrfs_set_backup_dev_root_gen(root_backup,
1939                                btrfs_header_generation(info->dev_root->node));
1940         btrfs_set_backup_dev_root_level(root_backup,
1941                                        btrfs_header_level(info->dev_root->node));
1942
1943         btrfs_set_backup_csum_root(root_backup, info->csum_root->node->start);
1944         btrfs_set_backup_csum_root_gen(root_backup,
1945                                btrfs_header_generation(info->csum_root->node));
1946         btrfs_set_backup_csum_root_level(root_backup,
1947                                btrfs_header_level(info->csum_root->node));
1948
1949         btrfs_set_backup_total_bytes(root_backup,
1950                              btrfs_super_total_bytes(info->super_copy));
1951         btrfs_set_backup_bytes_used(root_backup,
1952                              btrfs_super_bytes_used(info->super_copy));
1953         btrfs_set_backup_num_devices(root_backup,
1954                              btrfs_super_num_devices(info->super_copy));
1955
1956         /*
1957          * if we don't copy this out to the super_copy, it won't get remembered
1958          * for the next commit
1959          */
1960         memcpy(&info->super_copy->super_roots,
1961                &info->super_for_commit->super_roots,
1962                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
1963 }
1964
1965 /*
1966  * this copies info out of the root backup array and back into
1967  * the in-memory super block.  It is meant to help iterate through
1968  * the array, so you send it the number of backups you've already
1969  * tried and the last backup index you used.
1970  *
1971  * this returns -1 when it has tried all the backups
1972  */
1973 static noinline int next_root_backup(struct btrfs_fs_info *info,
1974                                      struct btrfs_super_block *super,
1975                                      int *num_backups_tried, int *backup_index)
1976 {
1977         struct btrfs_root_backup *root_backup;
1978         int newest = *backup_index;
1979
1980         if (*num_backups_tried == 0) {
1981                 u64 gen = btrfs_super_generation(super);
1982
1983                 newest = find_newest_super_backup(info, gen);
1984                 if (newest == -1)
1985                         return -1;
1986
1987                 *backup_index = newest;
1988                 *num_backups_tried = 1;
1989         } else if (*num_backups_tried == BTRFS_NUM_BACKUP_ROOTS) {
1990                 /* we've tried all the backups, all done */
1991                 return -1;
1992         } else {
1993                 /* jump to the next oldest backup */
1994                 newest = (*backup_index + BTRFS_NUM_BACKUP_ROOTS - 1) %
1995                         BTRFS_NUM_BACKUP_ROOTS;
1996                 *backup_index = newest;
1997                 *num_backups_tried += 1;
1998         }
1999         root_backup = super->super_roots + newest;
2000
2001         btrfs_set_super_generation(super,
2002                                    btrfs_backup_tree_root_gen(root_backup));
2003         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
2004         btrfs_set_super_root_level(super,
2005                                    btrfs_backup_tree_root_level(root_backup));
2006         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
2007
2008         /*
2009          * fixme: the total bytes and num_devices need to match or we should
2010          * need a fsck
2011          */
2012         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
2013         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
2014         return 0;
2015 }
2016
2017 /* helper to cleanup workers */
2018 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
2019 {
2020         btrfs_destroy_workqueue(fs_info->fixup_workers);
2021         btrfs_destroy_workqueue(fs_info->delalloc_workers);
2022         btrfs_destroy_workqueue(fs_info->workers);
2023         btrfs_destroy_workqueue(fs_info->endio_workers);
2024         btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
2025         btrfs_destroy_workqueue(fs_info->endio_repair_workers);
2026         btrfs_destroy_workqueue(fs_info->rmw_workers);
2027         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2028         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2029         btrfs_destroy_workqueue(fs_info->submit_workers);
2030         btrfs_destroy_workqueue(fs_info->delayed_workers);
2031         btrfs_destroy_workqueue(fs_info->caching_workers);
2032         btrfs_destroy_workqueue(fs_info->readahead_workers);
2033         btrfs_destroy_workqueue(fs_info->flush_workers);
2034         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2035         btrfs_destroy_workqueue(fs_info->extent_workers);
2036         /*
2037          * Now that all other work queues are destroyed, we can safely destroy
2038          * the queues used for metadata I/O, since tasks from those other work
2039          * queues can do metadata I/O operations.
2040          */
2041         btrfs_destroy_workqueue(fs_info->endio_meta_workers);
2042         btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
2043 }
2044
2045 static void free_root_extent_buffers(struct btrfs_root *root)
2046 {
2047         if (root) {
2048                 free_extent_buffer(root->node);
2049                 free_extent_buffer(root->commit_root);
2050                 root->node = NULL;
2051                 root->commit_root = NULL;
2052         }
2053 }
2054
2055 /* helper to cleanup tree roots */
2056 static void free_root_pointers(struct btrfs_fs_info *info, int chunk_root)
2057 {
2058         free_root_extent_buffers(info->tree_root);
2059
2060         free_root_extent_buffers(info->dev_root);
2061         free_root_extent_buffers(info->extent_root);
2062         free_root_extent_buffers(info->csum_root);
2063         free_root_extent_buffers(info->quota_root);
2064         free_root_extent_buffers(info->uuid_root);
2065         if (chunk_root)
2066                 free_root_extent_buffers(info->chunk_root);
2067         free_root_extent_buffers(info->free_space_root);
2068 }
2069
2070 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2071 {
2072         int ret;
2073         struct btrfs_root *gang[8];
2074         int i;
2075
2076         while (!list_empty(&fs_info->dead_roots)) {
2077                 gang[0] = list_entry(fs_info->dead_roots.next,
2078                                      struct btrfs_root, root_list);
2079                 list_del(&gang[0]->root_list);
2080
2081                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) {
2082                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2083                 } else {
2084                         free_extent_buffer(gang[0]->node);
2085                         free_extent_buffer(gang[0]->commit_root);
2086                         btrfs_put_fs_root(gang[0]);
2087                 }
2088         }
2089
2090         while (1) {
2091                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2092                                              (void **)gang, 0,
2093                                              ARRAY_SIZE(gang));
2094                 if (!ret)
2095                         break;
2096                 for (i = 0; i < ret; i++)
2097                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2098         }
2099
2100         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
2101                 btrfs_free_log_root_tree(NULL, fs_info);
2102                 btrfs_destroy_pinned_extent(fs_info, fs_info->pinned_extents);
2103         }
2104 }
2105
2106 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2107 {
2108         mutex_init(&fs_info->scrub_lock);
2109         atomic_set(&fs_info->scrubs_running, 0);
2110         atomic_set(&fs_info->scrub_pause_req, 0);
2111         atomic_set(&fs_info->scrubs_paused, 0);
2112         atomic_set(&fs_info->scrub_cancel_req, 0);
2113         init_waitqueue_head(&fs_info->scrub_pause_wait);
2114         fs_info->scrub_workers_refcnt = 0;
2115 }
2116
2117 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2118 {
2119         spin_lock_init(&fs_info->balance_lock);
2120         mutex_init(&fs_info->balance_mutex);
2121         atomic_set(&fs_info->balance_pause_req, 0);
2122         atomic_set(&fs_info->balance_cancel_req, 0);
2123         fs_info->balance_ctl = NULL;
2124         init_waitqueue_head(&fs_info->balance_wait_q);
2125 }
2126
2127 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2128 {
2129         struct inode *inode = fs_info->btree_inode;
2130
2131         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2132         set_nlink(inode, 1);
2133         /*
2134          * we set the i_size on the btree inode to the max possible int.
2135          * the real end of the address space is determined by all of
2136          * the devices in the system
2137          */
2138         inode->i_size = OFFSET_MAX;
2139         inode->i_mapping->a_ops = &btree_aops;
2140
2141         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2142         extent_io_tree_init(&BTRFS_I(inode)->io_tree, inode);
2143         BTRFS_I(inode)->io_tree.track_uptodate = 0;
2144         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2145
2146         BTRFS_I(inode)->io_tree.ops = &btree_extent_io_ops;
2147
2148         BTRFS_I(inode)->root = fs_info->tree_root;
2149         memset(&BTRFS_I(inode)->location, 0, sizeof(struct btrfs_key));
2150         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2151         btrfs_insert_inode_hash(inode);
2152 }
2153
2154 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2155 {
2156         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2157         rwlock_init(&fs_info->dev_replace.lock);
2158         atomic_set(&fs_info->dev_replace.blocking_readers, 0);
2159         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2160         init_waitqueue_head(&fs_info->dev_replace.read_lock_wq);
2161 }
2162
2163 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2164 {
2165         spin_lock_init(&fs_info->qgroup_lock);
2166         mutex_init(&fs_info->qgroup_ioctl_lock);
2167         fs_info->qgroup_tree = RB_ROOT;
2168         fs_info->qgroup_op_tree = RB_ROOT;
2169         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2170         fs_info->qgroup_seq = 1;
2171         fs_info->qgroup_ulist = NULL;
2172         fs_info->qgroup_rescan_running = false;
2173         mutex_init(&fs_info->qgroup_rescan_lock);
2174 }
2175
2176 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info,
2177                 struct btrfs_fs_devices *fs_devices)
2178 {
2179         u32 max_active = fs_info->thread_pool_size;
2180         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2181
2182         fs_info->workers =
2183                 btrfs_alloc_workqueue(fs_info, "worker",
2184                                       flags | WQ_HIGHPRI, max_active, 16);
2185
2186         fs_info->delalloc_workers =
2187                 btrfs_alloc_workqueue(fs_info, "delalloc",
2188                                       flags, max_active, 2);
2189
2190         fs_info->flush_workers =
2191                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2192                                       flags, max_active, 0);
2193
2194         fs_info->caching_workers =
2195                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2196
2197         /*
2198          * a higher idle thresh on the submit workers makes it much more
2199          * likely that bios will be send down in a sane order to the
2200          * devices
2201          */
2202         fs_info->submit_workers =
2203                 btrfs_alloc_workqueue(fs_info, "submit", flags,
2204                                       min_t(u64, fs_devices->num_devices,
2205                                             max_active), 64);
2206
2207         fs_info->fixup_workers =
2208                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2209
2210         /*
2211          * endios are largely parallel and should have a very
2212          * low idle thresh
2213          */
2214         fs_info->endio_workers =
2215                 btrfs_alloc_workqueue(fs_info, "endio", flags, max_active, 4);
2216         fs_info->endio_meta_workers =
2217                 btrfs_alloc_workqueue(fs_info, "endio-meta", flags,
2218                                       max_active, 4);
2219         fs_info->endio_meta_write_workers =
2220                 btrfs_alloc_workqueue(fs_info, "endio-meta-write", flags,
2221                                       max_active, 2);
2222         fs_info->endio_raid56_workers =
2223                 btrfs_alloc_workqueue(fs_info, "endio-raid56", flags,
2224                                       max_active, 4);
2225         fs_info->endio_repair_workers =
2226                 btrfs_alloc_workqueue(fs_info, "endio-repair", flags, 1, 0);
2227         fs_info->rmw_workers =
2228                 btrfs_alloc_workqueue(fs_info, "rmw", flags, max_active, 2);
2229         fs_info->endio_write_workers =
2230                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2231                                       max_active, 2);
2232         fs_info->endio_freespace_worker =
2233                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2234                                       max_active, 0);
2235         fs_info->delayed_workers =
2236                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2237                                       max_active, 0);
2238         fs_info->readahead_workers =
2239                 btrfs_alloc_workqueue(fs_info, "readahead", flags,
2240                                       max_active, 2);
2241         fs_info->qgroup_rescan_workers =
2242                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2243         fs_info->extent_workers =
2244                 btrfs_alloc_workqueue(fs_info, "extent-refs", flags,
2245                                       min_t(u64, fs_devices->num_devices,
2246                                             max_active), 8);
2247
2248         if (!(fs_info->workers && fs_info->delalloc_workers &&
2249               fs_info->submit_workers && fs_info->flush_workers &&
2250               fs_info->endio_workers && fs_info->endio_meta_workers &&
2251               fs_info->endio_meta_write_workers &&
2252               fs_info->endio_repair_workers &&
2253               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2254               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2255               fs_info->caching_workers && fs_info->readahead_workers &&
2256               fs_info->fixup_workers && fs_info->delayed_workers &&
2257               fs_info->extent_workers &&
2258               fs_info->qgroup_rescan_workers)) {
2259                 return -ENOMEM;
2260         }
2261
2262         return 0;
2263 }
2264
2265 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2266                             struct btrfs_fs_devices *fs_devices)
2267 {
2268         int ret;
2269         struct btrfs_root *log_tree_root;
2270         struct btrfs_super_block *disk_super = fs_info->super_copy;
2271         u64 bytenr = btrfs_super_log_root(disk_super);
2272         int level = btrfs_super_log_root_level(disk_super);
2273
2274         if (fs_devices->rw_devices == 0) {
2275                 btrfs_warn(fs_info, "log replay required on RO media");
2276                 return -EIO;
2277         }
2278
2279         log_tree_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2280         if (!log_tree_root)
2281                 return -ENOMEM;
2282
2283         __setup_root(log_tree_root, fs_info, BTRFS_TREE_LOG_OBJECTID);
2284
2285         log_tree_root->node = read_tree_block(fs_info, bytenr,
2286                                               fs_info->generation + 1,
2287                                               level, NULL);
2288         if (IS_ERR(log_tree_root->node)) {
2289                 btrfs_warn(fs_info, "failed to read log tree");
2290                 ret = PTR_ERR(log_tree_root->node);
2291                 kfree(log_tree_root);
2292                 return ret;
2293         } else if (!extent_buffer_uptodate(log_tree_root->node)) {
2294                 btrfs_err(fs_info, "failed to read log tree");
2295                 free_extent_buffer(log_tree_root->node);
2296                 kfree(log_tree_root);
2297                 return -EIO;
2298         }
2299         /* returns with log_tree_root freed on success */
2300         ret = btrfs_recover_log_trees(log_tree_root);
2301         if (ret) {
2302                 btrfs_handle_fs_error(fs_info, ret,
2303                                       "Failed to recover log tree");
2304                 free_extent_buffer(log_tree_root->node);
2305                 kfree(log_tree_root);
2306                 return ret;
2307         }
2308
2309         if (sb_rdonly(fs_info->sb)) {
2310                 ret = btrfs_commit_super(fs_info);
2311                 if (ret)
2312                         return ret;
2313         }
2314
2315         return 0;
2316 }
2317
2318 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2319 {
2320         struct btrfs_root *tree_root = fs_info->tree_root;
2321         struct btrfs_root *root;
2322         struct btrfs_key location;
2323         int ret;
2324
2325         BUG_ON(!fs_info->tree_root);
2326
2327         location.objectid = BTRFS_EXTENT_TREE_OBJECTID;
2328         location.type = BTRFS_ROOT_ITEM_KEY;
2329         location.offset = 0;
2330
2331         root = btrfs_read_tree_root(tree_root, &location);
2332         if (IS_ERR(root)) {
2333                 ret = PTR_ERR(root);
2334                 goto out;
2335         }
2336         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2337         fs_info->extent_root = root;
2338
2339         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2340         root = btrfs_read_tree_root(tree_root, &location);
2341         if (IS_ERR(root)) {
2342                 ret = PTR_ERR(root);
2343                 goto out;
2344         }
2345         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2346         fs_info->dev_root = root;
2347         btrfs_init_devices_late(fs_info);
2348
2349         location.objectid = BTRFS_CSUM_TREE_OBJECTID;
2350         root = btrfs_read_tree_root(tree_root, &location);
2351         if (IS_ERR(root)) {
2352                 ret = PTR_ERR(root);
2353                 goto out;
2354         }
2355         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2356         fs_info->csum_root = root;
2357
2358         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2359         root = btrfs_read_tree_root(tree_root, &location);
2360         if (!IS_ERR(root)) {
2361                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2362                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2363                 fs_info->quota_root = root;
2364         }
2365
2366         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2367         root = btrfs_read_tree_root(tree_root, &location);
2368         if (IS_ERR(root)) {
2369                 ret = PTR_ERR(root);
2370                 if (ret != -ENOENT)
2371                         goto out;
2372         } else {
2373                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2374                 fs_info->uuid_root = root;
2375         }
2376
2377         if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
2378                 location.objectid = BTRFS_FREE_SPACE_TREE_OBJECTID;
2379                 root = btrfs_read_tree_root(tree_root, &location);
2380                 if (IS_ERR(root)) {
2381                         ret = PTR_ERR(root);
2382                         goto out;
2383                 }
2384                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2385                 fs_info->free_space_root = root;
2386         }
2387
2388         return 0;
2389 out:
2390         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2391                    location.objectid, ret);
2392         return ret;
2393 }
2394
2395 /*
2396  * Real super block validation
2397  * NOTE: super csum type and incompat features will not be checked here.
2398  *
2399  * @sb:         super block to check
2400  * @mirror_num: the super block number to check its bytenr:
2401  *              0       the primary (1st) sb
2402  *              1, 2    2nd and 3rd backup copy
2403  *             -1       skip bytenr check
2404  */
2405 static int validate_super(struct btrfs_fs_info *fs_info,
2406                             struct btrfs_super_block *sb, int mirror_num)
2407 {
2408         u64 nodesize = btrfs_super_nodesize(sb);
2409         u64 sectorsize = btrfs_super_sectorsize(sb);
2410         int ret = 0;
2411
2412         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2413                 btrfs_err(fs_info, "no valid FS found");
2414                 ret = -EINVAL;
2415         }
2416         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2417                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2418                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2419                 ret = -EINVAL;
2420         }
2421         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2422                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2423                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2424                 ret = -EINVAL;
2425         }
2426         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2427                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2428                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2429                 ret = -EINVAL;
2430         }
2431         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2432                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2433                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2434                 ret = -EINVAL;
2435         }
2436
2437         /*
2438          * Check sectorsize and nodesize first, other check will need it.
2439          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2440          */
2441         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2442             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2443                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2444                 ret = -EINVAL;
2445         }
2446         /* Only PAGE SIZE is supported yet */
2447         if (sectorsize != PAGE_SIZE) {
2448                 btrfs_err(fs_info,
2449                         "sectorsize %llu not supported yet, only support %lu",
2450                         sectorsize, PAGE_SIZE);
2451                 ret = -EINVAL;
2452         }
2453         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2454             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2455                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2456                 ret = -EINVAL;
2457         }
2458         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2459                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2460                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2461                 ret = -EINVAL;
2462         }
2463
2464         /* Root alignment check */
2465         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2466                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2467                            btrfs_super_root(sb));
2468                 ret = -EINVAL;
2469         }
2470         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2471                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2472                            btrfs_super_chunk_root(sb));
2473                 ret = -EINVAL;
2474         }
2475         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2476                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2477                            btrfs_super_log_root(sb));
2478                 ret = -EINVAL;
2479         }
2480
2481         if (memcmp(fs_info->fsid, sb->dev_item.fsid, BTRFS_FSID_SIZE) != 0) {
2482                 btrfs_err(fs_info,
2483                            "dev_item UUID does not match fsid: %pU != %pU",
2484                            fs_info->fsid, sb->dev_item.fsid);
2485                 ret = -EINVAL;
2486         }
2487
2488         /*
2489          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2490          * done later
2491          */
2492         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2493                 btrfs_err(fs_info, "bytes_used is too small %llu",
2494                           btrfs_super_bytes_used(sb));
2495                 ret = -EINVAL;
2496         }
2497         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2498                 btrfs_err(fs_info, "invalid stripesize %u",
2499                           btrfs_super_stripesize(sb));
2500                 ret = -EINVAL;
2501         }
2502         if (btrfs_super_num_devices(sb) > (1UL << 31))
2503                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2504                            btrfs_super_num_devices(sb));
2505         if (btrfs_super_num_devices(sb) == 0) {
2506                 btrfs_err(fs_info, "number of devices is 0");
2507                 ret = -EINVAL;
2508         }
2509
2510         if (mirror_num >= 0 &&
2511             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2512                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2513                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2514                 ret = -EINVAL;
2515         }
2516
2517         /*
2518          * Obvious sys_chunk_array corruptions, it must hold at least one key
2519          * and one chunk
2520          */
2521         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2522                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2523                           btrfs_super_sys_array_size(sb),
2524                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2525                 ret = -EINVAL;
2526         }
2527         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2528                         + sizeof(struct btrfs_chunk)) {
2529                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2530                           btrfs_super_sys_array_size(sb),
2531                           sizeof(struct btrfs_disk_key)
2532                           + sizeof(struct btrfs_chunk));
2533                 ret = -EINVAL;
2534         }
2535
2536         /*
2537          * The generation is a global counter, we'll trust it more than the others
2538          * but it's still possible that it's the one that's wrong.
2539          */
2540         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2541                 btrfs_warn(fs_info,
2542                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2543                         btrfs_super_generation(sb),
2544                         btrfs_super_chunk_root_generation(sb));
2545         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2546             && btrfs_super_cache_generation(sb) != (u64)-1)
2547                 btrfs_warn(fs_info,
2548                         "suspicious: generation < cache_generation: %llu < %llu",
2549                         btrfs_super_generation(sb),
2550                         btrfs_super_cache_generation(sb));
2551
2552         return ret;
2553 }
2554
2555 /*
2556  * Validation of super block at mount time.
2557  * Some checks already done early at mount time, like csum type and incompat
2558  * flags will be skipped.
2559  */
2560 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2561 {
2562         return validate_super(fs_info, fs_info->super_copy, 0);
2563 }
2564
2565 /*
2566  * Validation of super block at write time.
2567  * Some checks like bytenr check will be skipped as their values will be
2568  * overwritten soon.
2569  * Extra checks like csum type and incompat flags will be done here.
2570  */
2571 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2572                                       struct btrfs_super_block *sb)
2573 {
2574         int ret;
2575
2576         ret = validate_super(fs_info, sb, -1);
2577         if (ret < 0)
2578                 goto out;
2579         if (btrfs_super_csum_type(sb) != BTRFS_CSUM_TYPE_CRC32) {
2580                 ret = -EUCLEAN;
2581                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2582                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2583                 goto out;
2584         }
2585         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2586                 ret = -EUCLEAN;
2587                 btrfs_err(fs_info,
2588                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2589                           btrfs_super_incompat_flags(sb),
2590                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2591                 goto out;
2592         }
2593 out:
2594         if (ret < 0)
2595                 btrfs_err(fs_info,
2596                 "super block corruption detected before writing it to disk");
2597         return ret;
2598 }
2599
2600 int open_ctree(struct super_block *sb,
2601                struct btrfs_fs_devices *fs_devices,
2602                char *options)
2603 {
2604         u32 sectorsize;
2605         u32 nodesize;
2606         u32 stripesize;
2607         u64 generation;
2608         u64 features;
2609         struct btrfs_key location;
2610         struct buffer_head *bh;
2611         struct btrfs_super_block *disk_super;
2612         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
2613         struct btrfs_root *tree_root;
2614         struct btrfs_root *chunk_root;
2615         int ret;
2616         int err = -EINVAL;
2617         int num_backups_tried = 0;
2618         int backup_index = 0;
2619         int clear_free_space_tree = 0;
2620         int level;
2621
2622         tree_root = fs_info->tree_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2623         chunk_root = fs_info->chunk_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2624         if (!tree_root || !chunk_root) {
2625                 err = -ENOMEM;
2626                 goto fail;
2627         }
2628
2629         ret = init_srcu_struct(&fs_info->subvol_srcu);
2630         if (ret) {
2631                 err = ret;
2632                 goto fail;
2633         }
2634
2635         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
2636         if (ret) {
2637                 err = ret;
2638                 goto fail_srcu;
2639         }
2640         fs_info->dirty_metadata_batch = PAGE_SIZE *
2641                                         (1 + ilog2(nr_cpu_ids));
2642
2643         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
2644         if (ret) {
2645                 err = ret;
2646                 goto fail_dirty_metadata_bytes;
2647         }
2648
2649         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
2650                         GFP_KERNEL);
2651         if (ret) {
2652                 err = ret;
2653                 goto fail_delalloc_bytes;
2654         }
2655
2656         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2657         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2658         INIT_LIST_HEAD(&fs_info->trans_list);
2659         INIT_LIST_HEAD(&fs_info->dead_roots);
2660         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2661         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2662         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2663         INIT_LIST_HEAD(&fs_info->pending_raid_kobjs);
2664         spin_lock_init(&fs_info->pending_raid_kobjs_lock);
2665         spin_lock_init(&fs_info->delalloc_root_lock);
2666         spin_lock_init(&fs_info->trans_lock);
2667         spin_lock_init(&fs_info->fs_roots_radix_lock);
2668         spin_lock_init(&fs_info->delayed_iput_lock);
2669         spin_lock_init(&fs_info->defrag_inodes_lock);
2670         spin_lock_init(&fs_info->tree_mod_seq_lock);
2671         spin_lock_init(&fs_info->super_lock);
2672         spin_lock_init(&fs_info->qgroup_op_lock);
2673         spin_lock_init(&fs_info->buffer_lock);
2674         spin_lock_init(&fs_info->unused_bgs_lock);
2675         rwlock_init(&fs_info->tree_mod_log_lock);
2676         mutex_init(&fs_info->unused_bg_unpin_mutex);
2677         mutex_init(&fs_info->delete_unused_bgs_mutex);
2678         mutex_init(&fs_info->reloc_mutex);
2679         mutex_init(&fs_info->delalloc_root_mutex);
2680         mutex_init(&fs_info->cleaner_delayed_iput_mutex);
2681         seqlock_init(&fs_info->profiles_lock);
2682
2683         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
2684         INIT_LIST_HEAD(&fs_info->space_info);
2685         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
2686         INIT_LIST_HEAD(&fs_info->unused_bgs);
2687         btrfs_mapping_init(&fs_info->mapping_tree);
2688         btrfs_init_block_rsv(&fs_info->global_block_rsv,
2689                              BTRFS_BLOCK_RSV_GLOBAL);
2690         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
2691         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
2692         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
2693         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
2694                              BTRFS_BLOCK_RSV_DELOPS);
2695         atomic_set(&fs_info->async_delalloc_pages, 0);
2696         atomic_set(&fs_info->defrag_running, 0);
2697         atomic_set(&fs_info->qgroup_op_seq, 0);
2698         atomic_set(&fs_info->reada_works_cnt, 0);
2699         atomic64_set(&fs_info->tree_mod_seq, 0);
2700         fs_info->sb = sb;
2701         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
2702         fs_info->metadata_ratio = 0;
2703         fs_info->defrag_inodes = RB_ROOT;
2704         atomic64_set(&fs_info->free_chunk_space, 0);
2705         fs_info->tree_mod_log = RB_ROOT;
2706         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
2707         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
2708         /* readahead state */
2709         INIT_RADIX_TREE(&fs_info->reada_tree, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
2710         spin_lock_init(&fs_info->reada_lock);
2711         btrfs_init_ref_verify(fs_info);
2712
2713         fs_info->thread_pool_size = min_t(unsigned long,
2714                                           num_online_cpus() + 2, 8);
2715
2716         INIT_LIST_HEAD(&fs_info->ordered_roots);
2717         spin_lock_init(&fs_info->ordered_root_lock);
2718
2719         fs_info->btree_inode = new_inode(sb);
2720         if (!fs_info->btree_inode) {
2721                 err = -ENOMEM;
2722                 goto fail_bio_counter;
2723         }
2724         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
2725
2726         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
2727                                         GFP_KERNEL);
2728         if (!fs_info->delayed_root) {
2729                 err = -ENOMEM;
2730                 goto fail_iput;
2731         }
2732         btrfs_init_delayed_root(fs_info->delayed_root);
2733
2734         btrfs_init_scrub(fs_info);
2735 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
2736         fs_info->check_integrity_print_mask = 0;
2737 #endif
2738         btrfs_init_balance(fs_info);
2739         btrfs_init_async_reclaim_work(&fs_info->async_reclaim_work);
2740
2741         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
2742         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
2743
2744         btrfs_init_btree_inode(fs_info);
2745
2746         spin_lock_init(&fs_info->block_group_cache_lock);
2747         fs_info->block_group_cache_tree = RB_ROOT;
2748         fs_info->first_logical_byte = (u64)-1;
2749
2750         extent_io_tree_init(&fs_info->freed_extents[0], NULL);
2751         extent_io_tree_init(&fs_info->freed_extents[1], NULL);
2752         fs_info->pinned_extents = &fs_info->freed_extents[0];
2753         set_bit(BTRFS_FS_BARRIER, &fs_info->flags);
2754
2755         mutex_init(&fs_info->ordered_operations_mutex);
2756         mutex_init(&fs_info->tree_log_mutex);
2757         mutex_init(&fs_info->chunk_mutex);
2758         mutex_init(&fs_info->transaction_kthread_mutex);
2759         mutex_init(&fs_info->cleaner_mutex);
2760         mutex_init(&fs_info->ro_block_group_mutex);
2761         init_rwsem(&fs_info->commit_root_sem);
2762         init_rwsem(&fs_info->cleanup_work_sem);
2763         init_rwsem(&fs_info->subvol_sem);
2764         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
2765
2766         btrfs_init_dev_replace_locks(fs_info);
2767         btrfs_init_qgroup(fs_info);
2768
2769         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
2770         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
2771
2772         init_waitqueue_head(&fs_info->transaction_throttle);
2773         init_waitqueue_head(&fs_info->transaction_wait);
2774         init_waitqueue_head(&fs_info->transaction_blocked_wait);
2775         init_waitqueue_head(&fs_info->async_submit_wait);
2776
2777         INIT_LIST_HEAD(&fs_info->pinned_chunks);
2778
2779         /* Usable values until the real ones are cached from the superblock */
2780         fs_info->nodesize = 4096;
2781         fs_info->sectorsize = 4096;
2782         fs_info->stripesize = 4096;
2783
2784         ret = btrfs_alloc_stripe_hash_table(fs_info);
2785         if (ret) {
2786                 err = ret;
2787                 goto fail_alloc;
2788         }
2789
2790         __setup_root(tree_root, fs_info, BTRFS_ROOT_TREE_OBJECTID);
2791
2792         invalidate_bdev(fs_devices->latest_bdev);
2793
2794         /*
2795          * Read super block and check the signature bytes only
2796          */
2797         bh = btrfs_read_dev_super(fs_devices->latest_bdev);
2798         if (IS_ERR(bh)) {
2799                 err = PTR_ERR(bh);
2800                 goto fail_alloc;
2801         }
2802
2803         /*
2804          * We want to check superblock checksum, the type is stored inside.
2805          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
2806          */
2807         if (btrfs_check_super_csum(fs_info, bh->b_data)) {
2808                 btrfs_err(fs_info, "superblock checksum mismatch");
2809                 err = -EINVAL;
2810                 brelse(bh);
2811                 goto fail_alloc;
2812         }
2813
2814         /*
2815          * super_copy is zeroed at allocation time and we never touch the
2816          * following bytes up to INFO_SIZE, the checksum is calculated from
2817          * the whole block of INFO_SIZE
2818          */
2819         memcpy(fs_info->super_copy, bh->b_data, sizeof(*fs_info->super_copy));
2820         memcpy(fs_info->super_for_commit, fs_info->super_copy,
2821                sizeof(*fs_info->super_for_commit));
2822         brelse(bh);
2823
2824         memcpy(fs_info->fsid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE);
2825
2826         ret = btrfs_validate_mount_super(fs_info);
2827         if (ret) {
2828                 btrfs_err(fs_info, "superblock contains fatal errors");
2829                 err = -EINVAL;
2830                 goto fail_alloc;
2831         }
2832
2833         disk_super = fs_info->super_copy;
2834         if (!btrfs_super_root(disk_super))
2835                 goto fail_alloc;
2836
2837         /* check FS state, whether FS is broken. */
2838         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
2839                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
2840
2841         /*
2842          * run through our array of backup supers and setup
2843          * our ring pointer to the oldest one
2844          */
2845         generation = btrfs_super_generation(disk_super);
2846         find_oldest_super_backup(fs_info, generation);
2847
2848         /*
2849          * In the long term, we'll store the compression type in the super
2850          * block, and it'll be used for per file compression control.
2851          */
2852         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
2853
2854         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
2855         if (ret) {
2856                 err = ret;
2857                 goto fail_alloc;
2858         }
2859
2860         features = btrfs_super_incompat_flags(disk_super) &
2861                 ~BTRFS_FEATURE_INCOMPAT_SUPP;
2862         if (features) {
2863                 btrfs_err(fs_info,
2864                     "cannot mount because of unsupported optional features (%llx)",
2865                     features);
2866                 err = -EINVAL;
2867                 goto fail_alloc;
2868         }
2869
2870         features = btrfs_super_incompat_flags(disk_super);
2871         features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
2872         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
2873                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
2874         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
2875                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
2876
2877         if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
2878                 btrfs_info(fs_info, "has skinny extents");
2879
2880         /*
2881          * flag our filesystem as having big metadata blocks if
2882          * they are bigger than the page size
2883          */
2884         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) {
2885                 if (!(features & BTRFS_FEATURE_INCOMPAT_BIG_METADATA))
2886                         btrfs_info(fs_info,
2887                                 "flagging fs with big metadata feature");
2888                 features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
2889         }
2890
2891         nodesize = btrfs_super_nodesize(disk_super);
2892         sectorsize = btrfs_super_sectorsize(disk_super);
2893         stripesize = sectorsize;
2894         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
2895         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
2896
2897         /* Cache block sizes */
2898         fs_info->nodesize = nodesize;
2899         fs_info->sectorsize = sectorsize;
2900         fs_info->stripesize = stripesize;
2901
2902         /*
2903          * mixed block groups end up with duplicate but slightly offset
2904          * extent buffers for the same range.  It leads to corruptions
2905          */
2906         if ((features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
2907             (sectorsize != nodesize)) {
2908                 btrfs_err(fs_info,
2909 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
2910                         nodesize, sectorsize);
2911                 goto fail_alloc;
2912         }
2913
2914         /*
2915          * Needn't use the lock because there is no other task which will
2916          * update the flag.
2917          */
2918         btrfs_set_super_incompat_flags(disk_super, features);
2919
2920         features = btrfs_super_compat_ro_flags(disk_super) &
2921                 ~BTRFS_FEATURE_COMPAT_RO_SUPP;
2922         if (!sb_rdonly(sb) && features) {
2923                 btrfs_err(fs_info,
2924         "cannot mount read-write because of unsupported optional features (%llx)",
2925                        features);
2926                 err = -EINVAL;
2927                 goto fail_alloc;
2928         }
2929
2930         ret = btrfs_init_workqueues(fs_info, fs_devices);
2931         if (ret) {
2932                 err = ret;
2933                 goto fail_sb_buffer;
2934         }
2935
2936         sb->s_bdi->congested_fn = btrfs_congested_fn;
2937         sb->s_bdi->congested_data = fs_info;
2938         sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK;
2939         sb->s_bdi->ra_pages = VM_MAX_READAHEAD * SZ_1K / PAGE_SIZE;
2940         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
2941         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
2942
2943         sb->s_blocksize = sectorsize;
2944         sb->s_blocksize_bits = blksize_bits(sectorsize);
2945         memcpy(&sb->s_uuid, fs_info->fsid, BTRFS_FSID_SIZE);
2946
2947         mutex_lock(&fs_info->chunk_mutex);
2948         ret = btrfs_read_sys_array(fs_info);
2949         mutex_unlock(&fs_info->chunk_mutex);
2950         if (ret) {
2951                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
2952                 goto fail_sb_buffer;
2953         }
2954
2955         generation = btrfs_super_chunk_root_generation(disk_super);
2956         level = btrfs_super_chunk_root_level(disk_super);
2957
2958         __setup_root(chunk_root, fs_info, BTRFS_CHUNK_TREE_OBJECTID);
2959
2960         chunk_root->node = read_tree_block(fs_info,
2961                                            btrfs_super_chunk_root(disk_super),
2962                                            generation, level, NULL);
2963         if (IS_ERR(chunk_root->node) ||
2964             !extent_buffer_uptodate(chunk_root->node)) {
2965                 btrfs_err(fs_info, "failed to read chunk root");
2966                 if (!IS_ERR(chunk_root->node))
2967                         free_extent_buffer(chunk_root->node);
2968                 chunk_root->node = NULL;
2969                 goto fail_tree_roots;
2970         }
2971         btrfs_set_root_node(&chunk_root->root_item, chunk_root->node);
2972         chunk_root->commit_root = btrfs_root_node(chunk_root);
2973
2974         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
2975            btrfs_header_chunk_tree_uuid(chunk_root->node), BTRFS_UUID_SIZE);
2976
2977         ret = btrfs_read_chunk_tree(fs_info);
2978         if (ret) {
2979                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
2980                 goto fail_tree_roots;
2981         }
2982
2983         /*
2984          * Keep the devid that is marked to be the target device for the
2985          * device replace procedure
2986          */
2987         btrfs_free_extra_devids(fs_devices, 0);
2988
2989         if (!fs_devices->latest_bdev) {
2990                 btrfs_err(fs_info, "failed to read devices");
2991                 goto fail_tree_roots;
2992         }
2993
2994 retry_root_backup:
2995         generation = btrfs_super_generation(disk_super);
2996         level = btrfs_super_root_level(disk_super);
2997
2998         tree_root->node = read_tree_block(fs_info,
2999                                           btrfs_super_root(disk_super),
3000                                           generation, level, NULL);
3001         if (IS_ERR(tree_root->node) ||
3002             !extent_buffer_uptodate(tree_root->node)) {
3003                 btrfs_warn(fs_info, "failed to read tree root");
3004                 if (!IS_ERR(tree_root->node))
3005                         free_extent_buffer(tree_root->node);
3006                 tree_root->node = NULL;
3007                 goto recovery_tree_root;
3008         }
3009
3010         btrfs_set_root_node(&tree_root->root_item, tree_root->node);
3011         tree_root->commit_root = btrfs_root_node(tree_root);
3012         btrfs_set_root_refs(&tree_root->root_item, 1);
3013
3014         mutex_lock(&tree_root->objectid_mutex);
3015         ret = btrfs_find_highest_objectid(tree_root,
3016                                         &tree_root->highest_objectid);
3017         if (ret) {
3018                 mutex_unlock(&tree_root->objectid_mutex);
3019                 goto recovery_tree_root;
3020         }
3021
3022         ASSERT(tree_root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
3023
3024         mutex_unlock(&tree_root->objectid_mutex);
3025
3026         ret = btrfs_read_roots(fs_info);
3027         if (ret)
3028                 goto recovery_tree_root;
3029
3030         fs_info->generation = generation;
3031         fs_info->last_trans_committed = generation;
3032
3033         ret = btrfs_verify_dev_extents(fs_info);
3034         if (ret) {
3035                 btrfs_err(fs_info,
3036                           "failed to verify dev extents against chunks: %d",
3037                           ret);
3038                 goto fail_block_groups;
3039         }
3040         ret = btrfs_recover_balance(fs_info);
3041         if (ret) {
3042                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3043                 goto fail_block_groups;
3044         }
3045
3046         ret = btrfs_init_dev_stats(fs_info);
3047         if (ret) {
3048                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3049                 goto fail_block_groups;
3050         }
3051
3052         ret = btrfs_init_dev_replace(fs_info);
3053         if (ret) {
3054                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3055                 goto fail_block_groups;
3056         }
3057
3058         btrfs_free_extra_devids(fs_devices, 1);
3059
3060         ret = btrfs_sysfs_add_fsid(fs_devices, NULL);
3061         if (ret) {
3062                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3063                                 ret);
3064                 goto fail_block_groups;
3065         }
3066
3067         ret = btrfs_sysfs_add_device(fs_devices);
3068         if (ret) {
3069                 btrfs_err(fs_info, "failed to init sysfs device interface: %d",
3070                                 ret);
3071                 goto fail_fsdev_sysfs;
3072         }
3073
3074         ret = btrfs_sysfs_add_mounted(fs_info);
3075         if (ret) {
3076                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3077                 goto fail_fsdev_sysfs;
3078         }
3079
3080         ret = btrfs_init_space_info(fs_info);
3081         if (ret) {
3082                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3083                 goto fail_sysfs;
3084         }
3085
3086         ret = btrfs_read_block_groups(fs_info);
3087         if (ret) {
3088                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3089                 goto fail_sysfs;
3090         }
3091
3092         if (!sb_rdonly(sb) && !btrfs_check_rw_degradable(fs_info, NULL)) {
3093                 btrfs_warn(fs_info,
3094                 "writeable mount is not allowed due to too many missing devices");
3095                 goto fail_sysfs;
3096         }
3097
3098         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, tree_root,
3099                                                "btrfs-cleaner");
3100         if (IS_ERR(fs_info->cleaner_kthread))
3101                 goto fail_sysfs;
3102
3103         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3104                                                    tree_root,
3105                                                    "btrfs-transaction");
3106         if (IS_ERR(fs_info->transaction_kthread))
3107                 goto fail_cleaner;
3108
3109         if (!btrfs_test_opt(fs_info, NOSSD) &&
3110             !fs_info->fs_devices->rotating) {
3111                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3112         }
3113
3114         /*
3115          * Mount does not set all options immediately, we can do it now and do
3116          * not have to wait for transaction commit
3117          */
3118         btrfs_apply_pending_changes(fs_info);
3119
3120 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3121         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3122                 ret = btrfsic_mount(fs_info, fs_devices,
3123                                     btrfs_test_opt(fs_info,
3124                                         CHECK_INTEGRITY_INCLUDING_EXTENT_DATA) ?
3125                                     1 : 0,
3126                                     fs_info->check_integrity_print_mask);
3127                 if (ret)
3128                         btrfs_warn(fs_info,
3129                                 "failed to initialize integrity check module: %d",
3130                                 ret);
3131         }
3132 #endif
3133         ret = btrfs_read_qgroup_config(fs_info);
3134         if (ret)
3135                 goto fail_trans_kthread;
3136
3137         if (btrfs_build_ref_tree(fs_info))
3138                 btrfs_err(fs_info, "couldn't build ref tree");
3139
3140         /* do not make disk changes in broken FS or nologreplay is given */
3141         if (btrfs_super_log_root(disk_super) != 0 &&
3142             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3143                 ret = btrfs_replay_log(fs_info, fs_devices);
3144                 if (ret) {
3145                         err = ret;
3146                         goto fail_qgroup;
3147                 }
3148         }
3149
3150         ret = btrfs_find_orphan_roots(fs_info);
3151         if (ret)
3152                 goto fail_qgroup;
3153
3154         if (!sb_rdonly(sb)) {
3155                 ret = btrfs_cleanup_fs_roots(fs_info);
3156                 if (ret)
3157                         goto fail_qgroup;
3158
3159                 mutex_lock(&fs_info->cleaner_mutex);
3160                 ret = btrfs_recover_relocation(tree_root);
3161                 mutex_unlock(&fs_info->cleaner_mutex);
3162                 if (ret < 0) {
3163                         btrfs_warn(fs_info, "failed to recover relocation: %d",
3164                                         ret);
3165                         err = -EINVAL;
3166                         goto fail_qgroup;
3167                 }
3168         }
3169
3170         location.objectid = BTRFS_FS_TREE_OBJECTID;
3171         location.type = BTRFS_ROOT_ITEM_KEY;
3172         location.offset = 0;
3173
3174         fs_info->fs_root = btrfs_read_fs_root_no_name(fs_info, &location);
3175         if (IS_ERR(fs_info->fs_root)) {
3176                 err = PTR_ERR(fs_info->fs_root);
3177                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3178                 goto fail_qgroup;
3179         }
3180
3181         if (sb_rdonly(sb))
3182                 return 0;
3183
3184         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3185             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3186                 clear_free_space_tree = 1;
3187         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3188                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3189                 btrfs_warn(fs_info, "free space tree is invalid");
3190                 clear_free_space_tree = 1;
3191         }
3192
3193         if (clear_free_space_tree) {
3194                 btrfs_info(fs_info, "clearing free space tree");
3195                 ret = btrfs_clear_free_space_tree(fs_info);
3196                 if (ret) {
3197                         btrfs_warn(fs_info,
3198                                    "failed to clear free space tree: %d", ret);
3199                         close_ctree(fs_info);
3200                         return ret;
3201                 }
3202         }
3203
3204         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3205             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3206                 btrfs_info(fs_info, "creating free space tree");
3207                 ret = btrfs_create_free_space_tree(fs_info);
3208                 if (ret) {
3209                         btrfs_warn(fs_info,
3210                                 "failed to create free space tree: %d", ret);
3211                         close_ctree(fs_info);
3212                         return ret;
3213                 }
3214         }
3215
3216         down_read(&fs_info->cleanup_work_sem);
3217         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3218             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3219                 up_read(&fs_info->cleanup_work_sem);
3220                 close_ctree(fs_info);
3221                 return ret;
3222         }
3223         up_read(&fs_info->cleanup_work_sem);
3224
3225         ret = btrfs_resume_balance_async(fs_info);
3226         if (ret) {
3227                 btrfs_warn(fs_info, "failed to resume balance: %d", ret);
3228                 close_ctree(fs_info);
3229                 return ret;
3230         }
3231
3232         ret = btrfs_resume_dev_replace_async(fs_info);
3233         if (ret) {
3234                 btrfs_warn(fs_info, "failed to resume device replace: %d", ret);
3235                 close_ctree(fs_info);
3236                 return ret;
3237         }
3238
3239         btrfs_qgroup_rescan_resume(fs_info);
3240
3241         if (!fs_info->uuid_root) {
3242                 btrfs_info(fs_info, "creating UUID tree");
3243                 ret = btrfs_create_uuid_tree(fs_info);
3244                 if (ret) {
3245                         btrfs_warn(fs_info,
3246                                 "failed to create the UUID tree: %d", ret);
3247                         close_ctree(fs_info);
3248                         return ret;
3249                 }
3250         } else if (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3251                    fs_info->generation !=
3252                                 btrfs_super_uuid_tree_generation(disk_super)) {
3253                 btrfs_info(fs_info, "checking UUID tree");
3254                 ret = btrfs_check_uuid_tree(fs_info);
3255                 if (ret) {
3256                         btrfs_warn(fs_info,
3257                                 "failed to check the UUID tree: %d", ret);
3258                         close_ctree(fs_info);
3259                         return ret;
3260                 }
3261         } else {
3262                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3263         }
3264         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3265
3266         /*
3267          * backuproot only affect mount behavior, and if open_ctree succeeded,
3268          * no need to keep the flag
3269          */
3270         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3271
3272         return 0;
3273
3274 fail_qgroup:
3275         btrfs_free_qgroup_config(fs_info);
3276 fail_trans_kthread:
3277         kthread_stop(fs_info->transaction_kthread);
3278         btrfs_cleanup_transaction(fs_info);
3279         btrfs_free_fs_roots(fs_info);
3280 fail_cleaner:
3281         kthread_stop(fs_info->cleaner_kthread);
3282
3283         /*
3284          * make sure we're done with the btree inode before we stop our
3285          * kthreads
3286          */
3287         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3288
3289 fail_sysfs:
3290         btrfs_sysfs_remove_mounted(fs_info);
3291
3292 fail_fsdev_sysfs:
3293         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3294
3295 fail_block_groups:
3296         btrfs_put_block_group_cache(fs_info);
3297
3298 fail_tree_roots:
3299         free_root_pointers(fs_info, 1);
3300         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3301
3302 fail_sb_buffer:
3303         btrfs_stop_all_workers(fs_info);
3304         btrfs_free_block_groups(fs_info);
3305 fail_alloc:
3306 fail_iput:
3307         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3308
3309         iput(fs_info->btree_inode);
3310 fail_bio_counter:
3311         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
3312 fail_delalloc_bytes:
3313         percpu_counter_destroy(&fs_info->delalloc_bytes);
3314 fail_dirty_metadata_bytes:
3315         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
3316 fail_srcu:
3317         cleanup_srcu_struct(&fs_info->subvol_srcu);
3318 fail:
3319         btrfs_free_stripe_hash_table(fs_info);
3320         btrfs_close_devices(fs_info->fs_devices);
3321         return err;
3322
3323 recovery_tree_root:
3324         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
3325                 goto fail_tree_roots;
3326
3327         free_root_pointers(fs_info, 0);
3328
3329         /* don't use the log in recovery mode, it won't be valid */
3330         btrfs_set_super_log_root(disk_super, 0);
3331
3332         /* we can't trust the free space cache either */
3333         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
3334
3335         ret = next_root_backup(fs_info, fs_info->super_copy,
3336                                &num_backups_tried, &backup_index);
3337         if (ret == -1)
3338                 goto fail_block_groups;
3339         goto retry_root_backup;
3340 }
3341 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3342
3343 static void btrfs_end_buffer_write_sync(struct buffer_head *bh, int uptodate)
3344 {
3345         if (uptodate) {
3346                 set_buffer_uptodate(bh);
3347         } else {
3348                 struct btrfs_device *device = (struct btrfs_device *)
3349                         bh->b_private;
3350
3351                 btrfs_warn_rl_in_rcu(device->fs_info,
3352                                 "lost page write due to IO error on %s",
3353                                           rcu_str_deref(device->name));
3354                 /* note, we don't set_buffer_write_io_error because we have
3355                  * our own ways of dealing with the IO errors
3356                  */
3357                 clear_buffer_uptodate(bh);
3358                 btrfs_dev_stat_inc_and_print(device, BTRFS_DEV_STAT_WRITE_ERRS);
3359         }
3360         unlock_buffer(bh);
3361         put_bh(bh);
3362 }
3363
3364 int btrfs_read_dev_one_super(struct block_device *bdev, int copy_num,
3365                         struct buffer_head **bh_ret)
3366 {
3367         struct buffer_head *bh;
3368         struct btrfs_super_block *super;
3369         u64 bytenr;
3370
3371         bytenr = btrfs_sb_offset(copy_num);
3372         if (bytenr + BTRFS_SUPER_INFO_SIZE >= i_size_read(bdev->bd_inode))
3373                 return -EINVAL;
3374
3375         bh = __bread(bdev, bytenr / BTRFS_BDEV_BLOCKSIZE, BTRFS_SUPER_INFO_SIZE);
3376         /*
3377          * If we fail to read from the underlying devices, as of now
3378          * the best option we have is to mark it EIO.
3379          */
3380         if (!bh)
3381                 return -EIO;
3382
3383         super = (struct btrfs_super_block *)bh->b_data;
3384         if (btrfs_super_bytenr(super) != bytenr ||
3385                     btrfs_super_magic(super) != BTRFS_MAGIC) {
3386                 brelse(bh);
3387                 return -EINVAL;
3388         }
3389
3390         *bh_ret = bh;
3391         return 0;
3392 }
3393
3394
3395 struct buffer_head *btrfs_read_dev_super(struct block_device *bdev)
3396 {
3397         struct buffer_head *bh;
3398         struct buffer_head *latest = NULL;
3399         struct btrfs_super_block *super;
3400         int i;
3401         u64 transid = 0;
3402         int ret = -EINVAL;
3403
3404         /* we would like to check all the supers, but that would make
3405          * a btrfs mount succeed after a mkfs from a different FS.
3406          * So, we need to add a special mount option to scan for
3407          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3408          */
3409         for (i = 0; i < 1; i++) {
3410                 ret = btrfs_read_dev_one_super(bdev, i, &bh);
3411                 if (ret)
3412                         continue;
3413
3414                 super = (struct btrfs_super_block *)bh->b_data;
3415
3416                 if (!latest || btrfs_super_generation(super) > transid) {
3417                         brelse(latest);
3418                         latest = bh;
3419                         transid = btrfs_super_generation(super);
3420                 } else {
3421                         brelse(bh);
3422                 }
3423         }
3424
3425         if (!latest)
3426                 return ERR_PTR(ret);
3427
3428         return latest;
3429 }
3430
3431 /*
3432  * Write superblock @sb to the @device. Do not wait for completion, all the
3433  * buffer heads we write are pinned.
3434  *
3435  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3436  * the expected device size at commit time. Note that max_mirrors must be
3437  * same for write and wait phases.
3438  *
3439  * Return number of errors when buffer head is not found or submission fails.
3440  */
3441 static int write_dev_supers(struct btrfs_device *device,
3442                             struct btrfs_super_block *sb, int max_mirrors)
3443 {
3444         struct buffer_head *bh;
3445         int i;
3446         int ret;
3447         int errors = 0;
3448         u32 crc;
3449         u64 bytenr;
3450         int op_flags;
3451
3452         if (max_mirrors == 0)
3453                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3454
3455         for (i = 0; i < max_mirrors; i++) {
3456                 bytenr = btrfs_sb_offset(i);
3457                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3458                     device->commit_total_bytes)
3459                         break;
3460
3461                 btrfs_set_super_bytenr(sb, bytenr);
3462
3463                 crc = ~(u32)0;
3464                 crc = btrfs_csum_data((const char *)sb + BTRFS_CSUM_SIZE, crc,
3465                                       BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
3466                 btrfs_csum_final(crc, sb->csum);
3467
3468                 /* One reference for us, and we leave it for the caller */
3469                 bh = __getblk(device->bdev, bytenr / BTRFS_BDEV_BLOCKSIZE,
3470                               BTRFS_SUPER_INFO_SIZE);
3471                 if (!bh) {
3472                         btrfs_err(device->fs_info,
3473                             "couldn't get super buffer head for bytenr %llu",
3474                             bytenr);
3475                         errors++;
3476                         continue;
3477                 }
3478
3479                 memcpy(bh->b_data, sb, BTRFS_SUPER_INFO_SIZE);
3480
3481                 /* one reference for submit_bh */
3482                 get_bh(bh);
3483
3484                 set_buffer_uptodate(bh);
3485                 lock_buffer(bh);
3486                 bh->b_end_io = btrfs_end_buffer_write_sync;
3487                 bh->b_private = device;
3488
3489                 /*
3490                  * we fua the first super.  The others we allow
3491                  * to go down lazy.
3492                  */
3493                 op_flags = REQ_SYNC | REQ_META | REQ_PRIO;
3494                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
3495                         op_flags |= REQ_FUA;
3496                 ret = btrfsic_submit_bh(REQ_OP_WRITE, op_flags, bh);
3497                 if (ret)
3498                         errors++;
3499         }
3500         return errors < i ? 0 : -1;
3501 }
3502
3503 /*
3504  * Wait for write completion of superblocks done by write_dev_supers,
3505  * @max_mirrors same for write and wait phases.
3506  *
3507  * Return number of errors when buffer head is not found or not marked up to
3508  * date.
3509  */
3510 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
3511 {
3512         struct buffer_head *bh;
3513         int i;
3514         int errors = 0;
3515         bool primary_failed = false;
3516         u64 bytenr;
3517
3518         if (max_mirrors == 0)
3519                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3520
3521         for (i = 0; i < max_mirrors; i++) {
3522                 bytenr = btrfs_sb_offset(i);
3523                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3524                     device->commit_total_bytes)
3525                         break;
3526
3527                 bh = __find_get_block(device->bdev,
3528                                       bytenr / BTRFS_BDEV_BLOCKSIZE,
3529                                       BTRFS_SUPER_INFO_SIZE);
3530                 if (!bh) {
3531                         errors++;
3532                         if (i == 0)
3533                                 primary_failed = true;
3534                         continue;
3535                 }
3536                 wait_on_buffer(bh);
3537                 if (!buffer_uptodate(bh)) {
3538                         errors++;
3539                         if (i == 0)
3540                                 primary_failed = true;
3541                 }
3542
3543                 /* drop our reference */
3544                 brelse(bh);
3545
3546                 /* drop the reference from the writing run */
3547                 brelse(bh);
3548         }
3549
3550         /* log error, force error return */
3551         if (primary_failed) {
3552                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
3553                           device->devid);
3554                 return -1;
3555         }
3556
3557         return errors < i ? 0 : -1;
3558 }
3559
3560 /*
3561  * endio for the write_dev_flush, this will wake anyone waiting
3562  * for the barrier when it is done
3563  */
3564 static void btrfs_end_empty_barrier(struct bio *bio)
3565 {
3566         complete(bio->bi_private);
3567 }
3568
3569 /*
3570  * Submit a flush request to the device if it supports it. Error handling is
3571  * done in the waiting counterpart.
3572  */
3573 static void write_dev_flush(struct btrfs_device *device)
3574 {
3575         struct request_queue *q = bdev_get_queue(device->bdev);
3576         struct bio *bio = device->flush_bio;
3577
3578         if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags))
3579                 return;
3580
3581         bio_reset(bio);
3582         bio->bi_end_io = btrfs_end_empty_barrier;
3583         bio_set_dev(bio, device->bdev);
3584         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH;
3585         init_completion(&device->flush_wait);
3586         bio->bi_private = &device->flush_wait;
3587
3588         btrfsic_submit_bio(bio);
3589         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
3590 }
3591
3592 /*
3593  * If the flush bio has been submitted by write_dev_flush, wait for it.
3594  */
3595 static blk_status_t wait_dev_flush(struct btrfs_device *device)
3596 {
3597         struct bio *bio = device->flush_bio;
3598
3599         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
3600                 return BLK_STS_OK;
3601
3602         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
3603         wait_for_completion_io(&device->flush_wait);
3604
3605         return bio->bi_status;
3606 }
3607
3608 static int check_barrier_error(struct btrfs_fs_info *fs_info)
3609 {
3610         if (!btrfs_check_rw_degradable(fs_info, NULL))
3611                 return -EIO;
3612         return 0;
3613 }
3614
3615 /*
3616  * send an empty flush down to each device in parallel,
3617  * then wait for them
3618  */
3619 static int barrier_all_devices(struct btrfs_fs_info *info)
3620 {
3621         struct list_head *head;
3622         struct btrfs_device *dev;
3623         int errors_wait = 0;
3624         blk_status_t ret;
3625
3626         lockdep_assert_held(&info->fs_devices->device_list_mutex);
3627         /* send down all the barriers */
3628         head = &info->fs_devices->devices;
3629         list_for_each_entry(dev, head, dev_list) {
3630                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
3631                         continue;
3632                 if (!dev->bdev)
3633                         continue;
3634                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3635                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3636                         continue;
3637
3638                 write_dev_flush(dev);
3639                 dev->last_flush_error = BLK_STS_OK;
3640         }
3641
3642         /* wait for all the barriers */
3643         list_for_each_entry(dev, head, dev_list) {
3644                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
3645                         continue;
3646                 if (!dev->bdev) {
3647                         errors_wait++;
3648                         continue;
3649                 }
3650                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3651                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3652                         continue;
3653
3654                 ret = wait_dev_flush(dev);
3655                 if (ret) {
3656                         dev->last_flush_error = ret;
3657                         btrfs_dev_stat_inc_and_print(dev,
3658                                         BTRFS_DEV_STAT_FLUSH_ERRS);
3659                         errors_wait++;
3660                 }
3661         }
3662
3663         if (errors_wait) {
3664                 /*
3665                  * At some point we need the status of all disks
3666                  * to arrive at the volume status. So error checking
3667                  * is being pushed to a separate loop.
3668                  */
3669                 return check_barrier_error(info);
3670         }
3671         return 0;
3672 }
3673
3674 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
3675 {
3676         int raid_type;
3677         int min_tolerated = INT_MAX;
3678
3679         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
3680             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
3681                 min_tolerated = min(min_tolerated,
3682                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
3683                                     tolerated_failures);
3684
3685         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
3686                 if (raid_type == BTRFS_RAID_SINGLE)
3687                         continue;
3688                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
3689                         continue;
3690                 min_tolerated = min(min_tolerated,
3691                                     btrfs_raid_array[raid_type].
3692                                     tolerated_failures);
3693         }
3694
3695         if (min_tolerated == INT_MAX) {
3696                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
3697                 min_tolerated = 0;
3698         }
3699
3700         return min_tolerated;
3701 }
3702
3703 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
3704 {
3705         struct list_head *head;
3706         struct btrfs_device *dev;
3707         struct btrfs_super_block *sb;
3708         struct btrfs_dev_item *dev_item;
3709         int ret;
3710         int do_barriers;
3711         int max_errors;
3712         int total_errors = 0;
3713         u64 flags;
3714
3715         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
3716
3717         /*
3718          * max_mirrors == 0 indicates we're from commit_transaction,
3719          * not from fsync where the tree roots in fs_info have not
3720          * been consistent on disk.
3721          */
3722         if (max_mirrors == 0)
3723                 backup_super_roots(fs_info);
3724
3725         sb = fs_info->super_for_commit;
3726         dev_item = &sb->dev_item;
3727
3728         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3729         head = &fs_info->fs_devices->devices;
3730         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
3731
3732         if (do_barriers) {
3733                 ret = barrier_all_devices(fs_info);
3734                 if (ret) {
3735                         mutex_unlock(
3736                                 &fs_info->fs_devices->device_list_mutex);
3737                         btrfs_handle_fs_error(fs_info, ret,
3738                                               "errors while submitting device barriers.");
3739                         return ret;
3740                 }
3741         }
3742
3743         list_for_each_entry(dev, head, dev_list) {
3744                 if (!dev->bdev) {
3745                         total_errors++;
3746                         continue;
3747                 }
3748                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3749                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3750                         continue;
3751
3752                 btrfs_set_stack_device_generation(dev_item, 0);
3753                 btrfs_set_stack_device_type(dev_item, dev->type);
3754                 btrfs_set_stack_device_id(dev_item, dev->devid);
3755                 btrfs_set_stack_device_total_bytes(dev_item,
3756                                                    dev->commit_total_bytes);
3757                 btrfs_set_stack_device_bytes_used(dev_item,
3758                                                   dev->commit_bytes_used);
3759                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
3760                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
3761                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
3762                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
3763                 memcpy(dev_item->fsid, dev->fs_devices->fsid, BTRFS_FSID_SIZE);
3764
3765                 flags = btrfs_super_flags(sb);
3766                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
3767
3768                 ret = btrfs_validate_write_super(fs_info, sb);
3769                 if (ret < 0) {
3770                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3771                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
3772                                 "unexpected superblock corruption detected");
3773                         return -EUCLEAN;
3774                 }
3775
3776                 ret = write_dev_supers(dev, sb, max_mirrors);
3777                 if (ret)
3778                         total_errors++;
3779         }
3780         if (total_errors > max_errors) {
3781                 btrfs_err(fs_info, "%d errors while writing supers",
3782                           total_errors);
3783                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3784
3785                 /* FUA is masked off if unsupported and can't be the reason */
3786                 btrfs_handle_fs_error(fs_info, -EIO,
3787                                       "%d errors while writing supers",
3788                                       total_errors);
3789                 return -EIO;
3790         }
3791
3792         total_errors = 0;
3793         list_for_each_entry(dev, head, dev_list) {
3794                 if (!dev->bdev)
3795                         continue;
3796                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3797                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3798                         continue;
3799
3800                 ret = wait_dev_supers(dev, max_mirrors);
3801                 if (ret)
3802                         total_errors++;
3803         }
3804         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3805         if (total_errors > max_errors) {
3806                 btrfs_handle_fs_error(fs_info, -EIO,
3807                                       "%d errors while writing supers",
3808                                       total_errors);
3809                 return -EIO;
3810         }
3811         return 0;
3812 }
3813
3814 /* Drop a fs root from the radix tree and free it. */
3815 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
3816                                   struct btrfs_root *root)
3817 {
3818         spin_lock(&fs_info->fs_roots_radix_lock);
3819         radix_tree_delete(&fs_info->fs_roots_radix,
3820                           (unsigned long)root->root_key.objectid);
3821         spin_unlock(&fs_info->fs_roots_radix_lock);
3822
3823         if (btrfs_root_refs(&root->root_item) == 0)
3824                 synchronize_srcu(&fs_info->subvol_srcu);
3825
3826         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
3827                 btrfs_free_log(NULL, root);
3828                 if (root->reloc_root) {
3829                         free_extent_buffer(root->reloc_root->node);
3830                         free_extent_buffer(root->reloc_root->commit_root);
3831                         btrfs_put_fs_root(root->reloc_root);
3832                         root->reloc_root = NULL;
3833                 }
3834         }
3835
3836         if (root->free_ino_pinned)
3837                 __btrfs_remove_free_space_cache(root->free_ino_pinned);
3838         if (root->free_ino_ctl)
3839                 __btrfs_remove_free_space_cache(root->free_ino_ctl);
3840         btrfs_free_fs_root(root);
3841 }
3842
3843 void btrfs_free_fs_root(struct btrfs_root *root)
3844 {
3845         iput(root->ino_cache_inode);
3846         WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
3847         if (root->anon_dev)
3848                 free_anon_bdev(root->anon_dev);
3849         if (root->subv_writers)
3850                 btrfs_free_subvolume_writers(root->subv_writers);
3851         free_extent_buffer(root->node);
3852         free_extent_buffer(root->commit_root);
3853         kfree(root->free_ino_ctl);
3854         kfree(root->free_ino_pinned);
3855         btrfs_put_fs_root(root);
3856 }
3857
3858 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
3859 {
3860         u64 root_objectid = 0;
3861         struct btrfs_root *gang[8];
3862         int i = 0;
3863         int err = 0;
3864         unsigned int ret = 0;
3865         int index;
3866
3867         while (1) {
3868                 index = srcu_read_lock(&fs_info->subvol_srcu);
3869                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
3870                                              (void **)gang, root_objectid,
3871                                              ARRAY_SIZE(gang));
3872                 if (!ret) {
3873                         srcu_read_unlock(&fs_info->subvol_srcu, index);
3874                         break;
3875                 }
3876                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
3877
3878                 for (i = 0; i < ret; i++) {
3879                         /* Avoid to grab roots in dead_roots */
3880                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
3881                                 gang[i] = NULL;
3882                                 continue;
3883                         }
3884                         /* grab all the search result for later use */
3885                         gang[i] = btrfs_grab_fs_root(gang[i]);
3886                 }
3887                 srcu_read_unlock(&fs_info->subvol_srcu, index);
3888
3889                 for (i = 0; i < ret; i++) {
3890                         if (!gang[i])
3891                                 continue;
3892                         root_objectid = gang[i]->root_key.objectid;
3893                         err = btrfs_orphan_cleanup(gang[i]);
3894                         if (err)
3895                                 break;
3896                         btrfs_put_fs_root(gang[i]);
3897                 }
3898                 root_objectid++;
3899         }
3900
3901         /* release the uncleaned roots due to error */
3902         for (; i < ret; i++) {
3903                 if (gang[i])
3904                         btrfs_put_fs_root(gang[i]);
3905         }
3906         return err;
3907 }
3908
3909 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
3910 {
3911         struct btrfs_root *root = fs_info->tree_root;
3912         struct btrfs_trans_handle *trans;
3913
3914         mutex_lock(&fs_info->cleaner_mutex);
3915         btrfs_run_delayed_iputs(fs_info);
3916         mutex_unlock(&fs_info->cleaner_mutex);
3917         wake_up_process(fs_info->cleaner_kthread);
3918
3919         /* wait until ongoing cleanup work done */
3920         down_write(&fs_info->cleanup_work_sem);
3921         up_write(&fs_info->cleanup_work_sem);
3922
3923         trans = btrfs_join_transaction(root);
3924         if (IS_ERR(trans))
3925                 return PTR_ERR(trans);
3926         return btrfs_commit_transaction(trans);
3927 }
3928
3929 void close_ctree(struct btrfs_fs_info *fs_info)
3930 {
3931         int ret;
3932
3933         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
3934
3935         /* wait for the qgroup rescan worker to stop */
3936         btrfs_qgroup_wait_for_completion(fs_info, false);
3937
3938         /* wait for the uuid_scan task to finish */
3939         down(&fs_info->uuid_tree_rescan_sem);
3940         /* avoid complains from lockdep et al., set sem back to initial state */
3941         up(&fs_info->uuid_tree_rescan_sem);
3942
3943         /* pause restriper - we want to resume on mount */
3944         btrfs_pause_balance(fs_info);
3945
3946         btrfs_dev_replace_suspend_for_unmount(fs_info);
3947
3948         btrfs_scrub_cancel(fs_info);
3949
3950         /* wait for any defraggers to finish */
3951         wait_event(fs_info->transaction_wait,
3952                    (atomic_read(&fs_info->defrag_running) == 0));
3953
3954         /* clear out the rbtree of defraggable inodes */
3955         btrfs_cleanup_defrag_inodes(fs_info);
3956
3957         cancel_work_sync(&fs_info->async_reclaim_work);
3958
3959         if (!sb_rdonly(fs_info->sb)) {
3960                 /*
3961                  * If the cleaner thread is stopped and there are
3962                  * block groups queued for removal, the deletion will be
3963                  * skipped when we quit the cleaner thread.
3964                  */
3965                 btrfs_delete_unused_bgs(fs_info);
3966
3967                 ret = btrfs_commit_super(fs_info);
3968                 if (ret)
3969                         btrfs_err(fs_info, "commit super ret %d", ret);
3970         }
3971
3972         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state) ||
3973             test_bit(BTRFS_FS_STATE_TRANS_ABORTED, &fs_info->fs_state))
3974                 btrfs_error_commit_super(fs_info);
3975
3976         kthread_stop(fs_info->transaction_kthread);
3977         kthread_stop(fs_info->cleaner_kthread);
3978
3979         ASSERT(list_empty(&fs_info->delayed_iputs));
3980         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
3981
3982         btrfs_free_qgroup_config(fs_info);
3983         ASSERT(list_empty(&fs_info->delalloc_roots));
3984
3985         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
3986                 btrfs_info(fs_info, "at unmount delalloc count %lld",
3987                        percpu_counter_sum(&fs_info->delalloc_bytes));
3988         }
3989
3990         btrfs_sysfs_remove_mounted(fs_info);
3991         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3992
3993         btrfs_free_fs_roots(fs_info);
3994
3995         btrfs_put_block_group_cache(fs_info);
3996
3997         /*
3998          * we must make sure there is not any read request to
3999          * submit after we stopping all workers.
4000          */
4001         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4002         btrfs_stop_all_workers(fs_info);
4003
4004         btrfs_free_block_groups(fs_info);
4005
4006         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4007         free_root_pointers(fs_info, 1);
4008
4009         iput(fs_info->btree_inode);
4010
4011 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4012         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4013                 btrfsic_unmount(fs_info->fs_devices);
4014 #endif
4015
4016         btrfs_close_devices(fs_info->fs_devices);
4017         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4018
4019         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
4020         percpu_counter_destroy(&fs_info->delalloc_bytes);
4021         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
4022         cleanup_srcu_struct(&fs_info->subvol_srcu);
4023
4024         btrfs_free_stripe_hash_table(fs_info);
4025         btrfs_free_ref_cache(fs_info);
4026
4027         while (!list_empty(&fs_info->pinned_chunks)) {
4028                 struct extent_map *em;
4029
4030                 em = list_first_entry(&fs_info->pinned_chunks,
4031                                       struct extent_map, list);
4032                 list_del_init(&em->list);
4033                 free_extent_map(em);
4034         }
4035 }
4036
4037 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4038                           int atomic)
4039 {
4040         int ret;
4041         struct inode *btree_inode = buf->pages[0]->mapping->host;
4042
4043         ret = extent_buffer_uptodate(buf);
4044         if (!ret)
4045                 return ret;
4046
4047         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4048                                     parent_transid, atomic);
4049         if (ret == -EAGAIN)
4050                 return ret;
4051         return !ret;
4052 }
4053
4054 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4055 {
4056         struct btrfs_fs_info *fs_info;
4057         struct btrfs_root *root;
4058         u64 transid = btrfs_header_generation(buf);
4059         int was_dirty;
4060
4061 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4062         /*
4063          * This is a fast path so only do this check if we have sanity tests
4064          * enabled.  Normal people shouldn't be using umapped buffers as dirty
4065          * outside of the sanity tests.
4066          */
4067         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4068                 return;
4069 #endif
4070         root = BTRFS_I(buf->pages[0]->mapping->host)->root;
4071         fs_info = root->fs_info;
4072         btrfs_assert_tree_locked(buf);
4073         if (transid != fs_info->generation)
4074                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4075                         buf->start, transid, fs_info->generation);
4076         was_dirty = set_extent_buffer_dirty(buf);
4077         if (!was_dirty)
4078                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4079                                          buf->len,
4080                                          fs_info->dirty_metadata_batch);
4081 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4082         /*
4083          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4084          * but item data not updated.
4085          * So here we should only check item pointers, not item data.
4086          */
4087         if (btrfs_header_level(buf) == 0 &&
4088             btrfs_check_leaf_relaxed(fs_info, buf)) {
4089                 btrfs_print_leaf(buf);
4090                 ASSERT(0);
4091         }
4092 #endif
4093 }
4094
4095 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4096                                         int flush_delayed)
4097 {
4098         /*
4099          * looks as though older kernels can get into trouble with
4100          * this code, they end up stuck in balance_dirty_pages forever
4101          */
4102         int ret;
4103
4104         if (current->flags & PF_MEMALLOC)
4105                 return;
4106
4107         if (flush_delayed)
4108                 btrfs_balance_delayed_items(fs_info);
4109
4110         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4111                                      BTRFS_DIRTY_METADATA_THRESH,
4112                                      fs_info->dirty_metadata_batch);
4113         if (ret > 0) {
4114                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4115         }
4116 }
4117
4118 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4119 {
4120         __btrfs_btree_balance_dirty(fs_info, 1);
4121 }
4122
4123 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4124 {
4125         __btrfs_btree_balance_dirty(fs_info, 0);
4126 }
4127
4128 int btrfs_read_buffer(struct extent_buffer *buf, u64 parent_transid, int level,
4129                       struct btrfs_key *first_key)
4130 {
4131         struct btrfs_root *root = BTRFS_I(buf->pages[0]->mapping->host)->root;
4132         struct btrfs_fs_info *fs_info = root->fs_info;
4133
4134         return btree_read_extent_buffer_pages(fs_info, buf, parent_transid,
4135                                               level, first_key);
4136 }
4137
4138 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4139 {
4140         /* cleanup FS via transaction */
4141         btrfs_cleanup_transaction(fs_info);
4142
4143         mutex_lock(&fs_info->cleaner_mutex);
4144         btrfs_run_delayed_iputs(fs_info);
4145         mutex_unlock(&fs_info->cleaner_mutex);
4146
4147         down_write(&fs_info->cleanup_work_sem);
4148         up_write(&fs_info->cleanup_work_sem);
4149 }
4150
4151 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4152 {
4153         struct btrfs_ordered_extent *ordered;
4154
4155         spin_lock(&root->ordered_extent_lock);
4156         /*
4157          * This will just short circuit the ordered completion stuff which will
4158          * make sure the ordered extent gets properly cleaned up.
4159          */
4160         list_for_each_entry(ordered, &root->ordered_extents,
4161                             root_extent_list)
4162                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4163         spin_unlock(&root->ordered_extent_lock);
4164 }
4165
4166 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4167 {
4168         struct btrfs_root *root;
4169         struct list_head splice;
4170
4171         INIT_LIST_HEAD(&splice);
4172
4173         spin_lock(&fs_info->ordered_root_lock);
4174         list_splice_init(&fs_info->ordered_roots, &splice);
4175         while (!list_empty(&splice)) {
4176                 root = list_first_entry(&splice, struct btrfs_root,
4177                                         ordered_root);
4178                 list_move_tail(&root->ordered_root,
4179                                &fs_info->ordered_roots);
4180
4181                 spin_unlock(&fs_info->ordered_root_lock);
4182                 btrfs_destroy_ordered_extents(root);
4183
4184                 cond_resched();
4185                 spin_lock(&fs_info->ordered_root_lock);
4186         }
4187         spin_unlock(&fs_info->ordered_root_lock);
4188 }
4189
4190 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4191                                       struct btrfs_fs_info *fs_info)
4192 {
4193         struct rb_node *node;
4194         struct btrfs_delayed_ref_root *delayed_refs;
4195         struct btrfs_delayed_ref_node *ref;
4196         int ret = 0;
4197
4198         delayed_refs = &trans->delayed_refs;
4199
4200         spin_lock(&delayed_refs->lock);
4201         if (atomic_read(&delayed_refs->num_entries) == 0) {
4202                 spin_unlock(&delayed_refs->lock);
4203                 btrfs_info(fs_info, "delayed_refs has NO entry");
4204                 return ret;
4205         }
4206
4207         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4208                 struct btrfs_delayed_ref_head *head;
4209                 struct rb_node *n;
4210                 bool pin_bytes = false;
4211
4212                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4213                                 href_node);
4214                 if (!mutex_trylock(&head->mutex)) {
4215                         refcount_inc(&head->refs);
4216                         spin_unlock(&delayed_refs->lock);
4217
4218                         mutex_lock(&head->mutex);
4219                         mutex_unlock(&head->mutex);
4220                         btrfs_put_delayed_ref_head(head);
4221                         spin_lock(&delayed_refs->lock);
4222                         continue;
4223                 }
4224                 spin_lock(&head->lock);
4225                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4226                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4227                                        ref_node);
4228                         ref->in_tree = 0;
4229                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4230                         RB_CLEAR_NODE(&ref->ref_node);
4231                         if (!list_empty(&ref->add_list))
4232                                 list_del(&ref->add_list);
4233                         atomic_dec(&delayed_refs->num_entries);
4234                         btrfs_put_delayed_ref(ref);
4235                 }
4236                 if (head->must_insert_reserved)
4237                         pin_bytes = true;
4238                 btrfs_free_delayed_extent_op(head->extent_op);
4239                 delayed_refs->num_heads--;
4240                 if (head->processing == 0)
4241                         delayed_refs->num_heads_ready--;
4242                 atomic_dec(&delayed_refs->num_entries);
4243                 rb_erase_cached(&head->href_node, &delayed_refs->href_root);
4244                 RB_CLEAR_NODE(&head->href_node);
4245                 spin_unlock(&head->lock);
4246                 spin_unlock(&delayed_refs->lock);
4247                 mutex_unlock(&head->mutex);
4248
4249                 if (pin_bytes)
4250                         btrfs_pin_extent(fs_info, head->bytenr,
4251                                          head->num_bytes, 1);
4252                 btrfs_put_delayed_ref_head(head);
4253                 cond_resched();
4254                 spin_lock(&delayed_refs->lock);
4255         }
4256
4257         spin_unlock(&delayed_refs->lock);
4258
4259         return ret;
4260 }
4261
4262 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4263 {
4264         struct btrfs_inode *btrfs_inode;
4265         struct list_head splice;
4266
4267         INIT_LIST_HEAD(&splice);
4268
4269         spin_lock(&root->delalloc_lock);
4270         list_splice_init(&root->delalloc_inodes, &splice);
4271
4272         while (!list_empty(&splice)) {
4273                 struct inode *inode = NULL;
4274                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4275                                                delalloc_inodes);
4276                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4277                 spin_unlock(&root->delalloc_lock);
4278
4279                 /*
4280                  * Make sure we get a live inode and that it'll not disappear
4281                  * meanwhile.
4282                  */
4283                 inode = igrab(&btrfs_inode->vfs_inode);
4284                 if (inode) {
4285                         invalidate_inode_pages2(inode->i_mapping);
4286                         iput(inode);
4287                 }
4288                 spin_lock(&root->delalloc_lock);
4289         }
4290         spin_unlock(&root->delalloc_lock);
4291 }
4292
4293 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
4294 {
4295         struct btrfs_root *root;
4296         struct list_head splice;
4297
4298         INIT_LIST_HEAD(&splice);
4299
4300         spin_lock(&fs_info->delalloc_root_lock);
4301         list_splice_init(&fs_info->delalloc_roots, &splice);
4302         while (!list_empty(&splice)) {
4303                 root = list_first_entry(&splice, struct btrfs_root,
4304                                          delalloc_root);
4305                 root = btrfs_grab_fs_root(root);
4306                 BUG_ON(!root);
4307                 spin_unlock(&fs_info->delalloc_root_lock);
4308
4309                 btrfs_destroy_delalloc_inodes(root);
4310                 btrfs_put_fs_root(root);
4311
4312                 spin_lock(&fs_info->delalloc_root_lock);
4313         }
4314         spin_unlock(&fs_info->delalloc_root_lock);
4315 }
4316
4317 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
4318                                         struct extent_io_tree *dirty_pages,
4319                                         int mark)
4320 {
4321         int ret;
4322         struct extent_buffer *eb;
4323         u64 start = 0;
4324         u64 end;
4325
4326         while (1) {
4327                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
4328                                             mark, NULL);
4329                 if (ret)
4330                         break;
4331
4332                 clear_extent_bits(dirty_pages, start, end, mark);
4333                 while (start <= end) {
4334                         eb = find_extent_buffer(fs_info, start);
4335                         start += fs_info->nodesize;
4336                         if (!eb)
4337                                 continue;
4338                         wait_on_extent_buffer_writeback(eb);
4339
4340                         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY,
4341                                                &eb->bflags))
4342                                 clear_extent_buffer_dirty(eb);
4343                         free_extent_buffer_stale(eb);
4344                 }
4345         }
4346
4347         return ret;
4348 }
4349
4350 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
4351                                        struct extent_io_tree *pinned_extents)
4352 {
4353         struct extent_io_tree *unpin;
4354         u64 start;
4355         u64 end;
4356         int ret;
4357         bool loop = true;
4358
4359         unpin = pinned_extents;
4360 again:
4361         while (1) {
4362                 ret = find_first_extent_bit(unpin, 0, &start, &end,
4363                                             EXTENT_DIRTY, NULL);
4364                 if (ret)
4365                         break;
4366
4367                 clear_extent_dirty(unpin, start, end);
4368                 btrfs_error_unpin_extent_range(fs_info, start, end);
4369                 cond_resched();
4370         }
4371
4372         if (loop) {
4373                 if (unpin == &fs_info->freed_extents[0])
4374                         unpin = &fs_info->freed_extents[1];
4375                 else
4376                         unpin = &fs_info->freed_extents[0];
4377                 loop = false;
4378                 goto again;
4379         }
4380
4381         return 0;
4382 }
4383
4384 static void btrfs_cleanup_bg_io(struct btrfs_block_group_cache *cache)
4385 {
4386         struct inode *inode;
4387
4388         inode = cache->io_ctl.inode;
4389         if (inode) {
4390                 invalidate_inode_pages2(inode->i_mapping);
4391                 BTRFS_I(inode)->generation = 0;
4392                 cache->io_ctl.inode = NULL;
4393                 iput(inode);
4394         }
4395         btrfs_put_block_group(cache);
4396 }
4397
4398 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
4399                              struct btrfs_fs_info *fs_info)
4400 {
4401         struct btrfs_block_group_cache *cache;
4402
4403         spin_lock(&cur_trans->dirty_bgs_lock);
4404         while (!list_empty(&cur_trans->dirty_bgs)) {
4405                 cache = list_first_entry(&cur_trans->dirty_bgs,
4406                                          struct btrfs_block_group_cache,
4407                                          dirty_list);
4408
4409                 if (!list_empty(&cache->io_list)) {
4410                         spin_unlock(&cur_trans->dirty_bgs_lock);
4411                         list_del_init(&cache->io_list);
4412                         btrfs_cleanup_bg_io(cache);
4413                         spin_lock(&cur_trans->dirty_bgs_lock);
4414                 }
4415
4416                 list_del_init(&cache->dirty_list);
4417                 spin_lock(&cache->lock);
4418                 cache->disk_cache_state = BTRFS_DC_ERROR;
4419                 spin_unlock(&cache->lock);
4420
4421                 spin_unlock(&cur_trans->dirty_bgs_lock);
4422                 btrfs_put_block_group(cache);
4423                 spin_lock(&cur_trans->dirty_bgs_lock);
4424         }
4425         spin_unlock(&cur_trans->dirty_bgs_lock);
4426
4427         /*
4428          * Refer to the definition of io_bgs member for details why it's safe
4429          * to use it without any locking
4430          */
4431         while (!list_empty(&cur_trans->io_bgs)) {
4432                 cache = list_first_entry(&cur_trans->io_bgs,
4433                                          struct btrfs_block_group_cache,
4434                                          io_list);
4435
4436                 list_del_init(&cache->io_list);
4437                 spin_lock(&cache->lock);
4438                 cache->disk_cache_state = BTRFS_DC_ERROR;
4439                 spin_unlock(&cache->lock);
4440                 btrfs_cleanup_bg_io(cache);
4441         }
4442 }
4443
4444 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
4445                                    struct btrfs_fs_info *fs_info)
4446 {
4447         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
4448         ASSERT(list_empty(&cur_trans->dirty_bgs));
4449         ASSERT(list_empty(&cur_trans->io_bgs));
4450
4451         btrfs_destroy_delayed_refs(cur_trans, fs_info);
4452
4453         cur_trans->state = TRANS_STATE_COMMIT_START;
4454         wake_up(&fs_info->transaction_blocked_wait);
4455
4456         cur_trans->state = TRANS_STATE_UNBLOCKED;
4457         wake_up(&fs_info->transaction_wait);
4458
4459         btrfs_destroy_delayed_inodes(fs_info);
4460         btrfs_assert_delayed_root_empty(fs_info);
4461
4462         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
4463                                      EXTENT_DIRTY);
4464         btrfs_destroy_pinned_extent(fs_info,
4465                                     fs_info->pinned_extents);
4466
4467         cur_trans->state =TRANS_STATE_COMPLETED;
4468         wake_up(&cur_trans->commit_wait);
4469 }
4470
4471 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
4472 {
4473         struct btrfs_transaction *t;
4474
4475         mutex_lock(&fs_info->transaction_kthread_mutex);
4476
4477         spin_lock(&fs_info->trans_lock);
4478         while (!list_empty(&fs_info->trans_list)) {
4479                 t = list_first_entry(&fs_info->trans_list,
4480                                      struct btrfs_transaction, list);
4481                 if (t->state >= TRANS_STATE_COMMIT_START) {
4482                         refcount_inc(&t->use_count);
4483                         spin_unlock(&fs_info->trans_lock);
4484                         btrfs_wait_for_commit(fs_info, t->transid);
4485                         btrfs_put_transaction(t);
4486                         spin_lock(&fs_info->trans_lock);
4487                         continue;
4488                 }
4489                 if (t == fs_info->running_transaction) {
4490                         t->state = TRANS_STATE_COMMIT_DOING;
4491                         spin_unlock(&fs_info->trans_lock);
4492                         /*
4493                          * We wait for 0 num_writers since we don't hold a trans
4494                          * handle open currently for this transaction.
4495                          */
4496                         wait_event(t->writer_wait,
4497                                    atomic_read(&t->num_writers) == 0);
4498                 } else {
4499                         spin_unlock(&fs_info->trans_lock);
4500                 }
4501                 btrfs_cleanup_one_transaction(t, fs_info);
4502
4503                 spin_lock(&fs_info->trans_lock);
4504                 if (t == fs_info->running_transaction)
4505                         fs_info->running_transaction = NULL;
4506                 list_del_init(&t->list);
4507                 spin_unlock(&fs_info->trans_lock);
4508
4509                 btrfs_put_transaction(t);
4510                 trace_btrfs_transaction_commit(fs_info->tree_root);
4511                 spin_lock(&fs_info->trans_lock);
4512         }
4513         spin_unlock(&fs_info->trans_lock);
4514         btrfs_destroy_all_ordered_extents(fs_info);
4515         btrfs_destroy_delayed_inodes(fs_info);
4516         btrfs_assert_delayed_root_empty(fs_info);
4517         btrfs_destroy_pinned_extent(fs_info, fs_info->pinned_extents);
4518         btrfs_destroy_all_delalloc_inodes(fs_info);
4519         mutex_unlock(&fs_info->transaction_kthread_mutex);
4520
4521         return 0;
4522 }
4523
4524 static const struct extent_io_ops btree_extent_io_ops = {
4525         /* mandatory callbacks */
4526         .submit_bio_hook = btree_submit_bio_hook,
4527         .readpage_end_io_hook = btree_readpage_end_io_hook,
4528         .readpage_io_failed_hook = btree_io_failed_hook,
4529
4530         /* optional callbacks */
4531 };