btrfs: remove unused member async_submit_bio::fs_info
[sfrench/cifs-2.6.git] / fs / btrfs / disk-io.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5
6 #include <linux/fs.h>
7 #include <linux/blkdev.h>
8 #include <linux/radix-tree.h>
9 #include <linux/writeback.h>
10 #include <linux/buffer_head.h>
11 #include <linux/workqueue.h>
12 #include <linux/kthread.h>
13 #include <linux/slab.h>
14 #include <linux/migrate.h>
15 #include <linux/ratelimit.h>
16 #include <linux/uuid.h>
17 #include <linux/semaphore.h>
18 #include <linux/error-injection.h>
19 #include <linux/crc32c.h>
20 #include <asm/unaligned.h>
21 #include "ctree.h"
22 #include "disk-io.h"
23 #include "transaction.h"
24 #include "btrfs_inode.h"
25 #include "volumes.h"
26 #include "print-tree.h"
27 #include "locking.h"
28 #include "tree-log.h"
29 #include "free-space-cache.h"
30 #include "free-space-tree.h"
31 #include "inode-map.h"
32 #include "check-integrity.h"
33 #include "rcu-string.h"
34 #include "dev-replace.h"
35 #include "raid56.h"
36 #include "sysfs.h"
37 #include "qgroup.h"
38 #include "compression.h"
39 #include "tree-checker.h"
40 #include "ref-verify.h"
41
42 #ifdef CONFIG_X86
43 #include <asm/cpufeature.h>
44 #endif
45
46 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
47                                  BTRFS_HEADER_FLAG_RELOC |\
48                                  BTRFS_SUPER_FLAG_ERROR |\
49                                  BTRFS_SUPER_FLAG_SEEDING |\
50                                  BTRFS_SUPER_FLAG_METADUMP |\
51                                  BTRFS_SUPER_FLAG_METADUMP_V2)
52
53 static const struct extent_io_ops btree_extent_io_ops;
54 static void end_workqueue_fn(struct btrfs_work *work);
55 static void free_fs_root(struct btrfs_root *root);
56 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
57 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
58                                       struct btrfs_fs_info *fs_info);
59 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
60 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
61                                         struct extent_io_tree *dirty_pages,
62                                         int mark);
63 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
64                                        struct extent_io_tree *pinned_extents);
65 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
66 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
67
68 /*
69  * btrfs_end_io_wq structs are used to do processing in task context when an IO
70  * is complete.  This is used during reads to verify checksums, and it is used
71  * by writes to insert metadata for new file extents after IO is complete.
72  */
73 struct btrfs_end_io_wq {
74         struct bio *bio;
75         bio_end_io_t *end_io;
76         void *private;
77         struct btrfs_fs_info *info;
78         blk_status_t status;
79         enum btrfs_wq_endio_type metadata;
80         struct btrfs_work work;
81 };
82
83 static struct kmem_cache *btrfs_end_io_wq_cache;
84
85 int __init btrfs_end_io_wq_init(void)
86 {
87         btrfs_end_io_wq_cache = kmem_cache_create("btrfs_end_io_wq",
88                                         sizeof(struct btrfs_end_io_wq),
89                                         0,
90                                         SLAB_MEM_SPREAD,
91                                         NULL);
92         if (!btrfs_end_io_wq_cache)
93                 return -ENOMEM;
94         return 0;
95 }
96
97 void __cold btrfs_end_io_wq_exit(void)
98 {
99         kmem_cache_destroy(btrfs_end_io_wq_cache);
100 }
101
102 /*
103  * async submit bios are used to offload expensive checksumming
104  * onto the worker threads.  They checksum file and metadata bios
105  * just before they are sent down the IO stack.
106  */
107 struct async_submit_bio {
108         void *private_data;
109         struct bio *bio;
110         extent_submit_bio_start_t *submit_bio_start;
111         extent_submit_bio_done_t *submit_bio_done;
112         int mirror_num;
113         unsigned long bio_flags;
114         /*
115          * bio_offset is optional, can be used if the pages in the bio
116          * can't tell us where in the file the bio should go
117          */
118         u64 bio_offset;
119         struct btrfs_work work;
120         blk_status_t status;
121 };
122
123 /*
124  * Lockdep class keys for extent_buffer->lock's in this root.  For a given
125  * eb, the lockdep key is determined by the btrfs_root it belongs to and
126  * the level the eb occupies in the tree.
127  *
128  * Different roots are used for different purposes and may nest inside each
129  * other and they require separate keysets.  As lockdep keys should be
130  * static, assign keysets according to the purpose of the root as indicated
131  * by btrfs_root->objectid.  This ensures that all special purpose roots
132  * have separate keysets.
133  *
134  * Lock-nesting across peer nodes is always done with the immediate parent
135  * node locked thus preventing deadlock.  As lockdep doesn't know this, use
136  * subclass to avoid triggering lockdep warning in such cases.
137  *
138  * The key is set by the readpage_end_io_hook after the buffer has passed
139  * csum validation but before the pages are unlocked.  It is also set by
140  * btrfs_init_new_buffer on freshly allocated blocks.
141  *
142  * We also add a check to make sure the highest level of the tree is the
143  * same as our lockdep setup here.  If BTRFS_MAX_LEVEL changes, this code
144  * needs update as well.
145  */
146 #ifdef CONFIG_DEBUG_LOCK_ALLOC
147 # if BTRFS_MAX_LEVEL != 8
148 #  error
149 # endif
150
151 static struct btrfs_lockdep_keyset {
152         u64                     id;             /* root objectid */
153         const char              *name_stem;     /* lock name stem */
154         char                    names[BTRFS_MAX_LEVEL + 1][20];
155         struct lock_class_key   keys[BTRFS_MAX_LEVEL + 1];
156 } btrfs_lockdep_keysets[] = {
157         { .id = BTRFS_ROOT_TREE_OBJECTID,       .name_stem = "root"     },
158         { .id = BTRFS_EXTENT_TREE_OBJECTID,     .name_stem = "extent"   },
159         { .id = BTRFS_CHUNK_TREE_OBJECTID,      .name_stem = "chunk"    },
160         { .id = BTRFS_DEV_TREE_OBJECTID,        .name_stem = "dev"      },
161         { .id = BTRFS_FS_TREE_OBJECTID,         .name_stem = "fs"       },
162         { .id = BTRFS_CSUM_TREE_OBJECTID,       .name_stem = "csum"     },
163         { .id = BTRFS_QUOTA_TREE_OBJECTID,      .name_stem = "quota"    },
164         { .id = BTRFS_TREE_LOG_OBJECTID,        .name_stem = "log"      },
165         { .id = BTRFS_TREE_RELOC_OBJECTID,      .name_stem = "treloc"   },
166         { .id = BTRFS_DATA_RELOC_TREE_OBJECTID, .name_stem = "dreloc"   },
167         { .id = BTRFS_UUID_TREE_OBJECTID,       .name_stem = "uuid"     },
168         { .id = BTRFS_FREE_SPACE_TREE_OBJECTID, .name_stem = "free-space" },
169         { .id = 0,                              .name_stem = "tree"     },
170 };
171
172 void __init btrfs_init_lockdep(void)
173 {
174         int i, j;
175
176         /* initialize lockdep class names */
177         for (i = 0; i < ARRAY_SIZE(btrfs_lockdep_keysets); i++) {
178                 struct btrfs_lockdep_keyset *ks = &btrfs_lockdep_keysets[i];
179
180                 for (j = 0; j < ARRAY_SIZE(ks->names); j++)
181                         snprintf(ks->names[j], sizeof(ks->names[j]),
182                                  "btrfs-%s-%02d", ks->name_stem, j);
183         }
184 }
185
186 void btrfs_set_buffer_lockdep_class(u64 objectid, struct extent_buffer *eb,
187                                     int level)
188 {
189         struct btrfs_lockdep_keyset *ks;
190
191         BUG_ON(level >= ARRAY_SIZE(ks->keys));
192
193         /* find the matching keyset, id 0 is the default entry */
194         for (ks = btrfs_lockdep_keysets; ks->id; ks++)
195                 if (ks->id == objectid)
196                         break;
197
198         lockdep_set_class_and_name(&eb->lock,
199                                    &ks->keys[level], ks->names[level]);
200 }
201
202 #endif
203
204 /*
205  * extents on the btree inode are pretty simple, there's one extent
206  * that covers the entire device
207  */
208 struct extent_map *btree_get_extent(struct btrfs_inode *inode,
209                 struct page *page, size_t pg_offset, u64 start, u64 len,
210                 int create)
211 {
212         struct btrfs_fs_info *fs_info = inode->root->fs_info;
213         struct extent_map_tree *em_tree = &inode->extent_tree;
214         struct extent_map *em;
215         int ret;
216
217         read_lock(&em_tree->lock);
218         em = lookup_extent_mapping(em_tree, start, len);
219         if (em) {
220                 em->bdev = fs_info->fs_devices->latest_bdev;
221                 read_unlock(&em_tree->lock);
222                 goto out;
223         }
224         read_unlock(&em_tree->lock);
225
226         em = alloc_extent_map();
227         if (!em) {
228                 em = ERR_PTR(-ENOMEM);
229                 goto out;
230         }
231         em->start = 0;
232         em->len = (u64)-1;
233         em->block_len = (u64)-1;
234         em->block_start = 0;
235         em->bdev = fs_info->fs_devices->latest_bdev;
236
237         write_lock(&em_tree->lock);
238         ret = add_extent_mapping(em_tree, em, 0);
239         if (ret == -EEXIST) {
240                 free_extent_map(em);
241                 em = lookup_extent_mapping(em_tree, start, len);
242                 if (!em)
243                         em = ERR_PTR(-EIO);
244         } else if (ret) {
245                 free_extent_map(em);
246                 em = ERR_PTR(ret);
247         }
248         write_unlock(&em_tree->lock);
249
250 out:
251         return em;
252 }
253
254 u32 btrfs_csum_data(const char *data, u32 seed, size_t len)
255 {
256         return crc32c(seed, data, len);
257 }
258
259 void btrfs_csum_final(u32 crc, u8 *result)
260 {
261         put_unaligned_le32(~crc, result);
262 }
263
264 /*
265  * compute the csum for a btree block, and either verify it or write it
266  * into the csum field of the block.
267  */
268 static int csum_tree_block(struct btrfs_fs_info *fs_info,
269                            struct extent_buffer *buf,
270                            int verify)
271 {
272         u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
273         char result[BTRFS_CSUM_SIZE];
274         unsigned long len;
275         unsigned long cur_len;
276         unsigned long offset = BTRFS_CSUM_SIZE;
277         char *kaddr;
278         unsigned long map_start;
279         unsigned long map_len;
280         int err;
281         u32 crc = ~(u32)0;
282
283         len = buf->len - offset;
284         while (len > 0) {
285                 err = map_private_extent_buffer(buf, offset, 32,
286                                         &kaddr, &map_start, &map_len);
287                 if (err)
288                         return err;
289                 cur_len = min(len, map_len - (offset - map_start));
290                 crc = btrfs_csum_data(kaddr + offset - map_start,
291                                       crc, cur_len);
292                 len -= cur_len;
293                 offset += cur_len;
294         }
295         memset(result, 0, BTRFS_CSUM_SIZE);
296
297         btrfs_csum_final(crc, result);
298
299         if (verify) {
300                 if (memcmp_extent_buffer(buf, result, 0, csum_size)) {
301                         u32 val;
302                         u32 found = 0;
303                         memcpy(&found, result, csum_size);
304
305                         read_extent_buffer(buf, &val, 0, csum_size);
306                         btrfs_warn_rl(fs_info,
307                                 "%s checksum verify failed on %llu wanted %X found %X level %d",
308                                 fs_info->sb->s_id, buf->start,
309                                 val, found, btrfs_header_level(buf));
310                         return -EUCLEAN;
311                 }
312         } else {
313                 write_extent_buffer(buf, result, 0, csum_size);
314         }
315
316         return 0;
317 }
318
319 /*
320  * we can't consider a given block up to date unless the transid of the
321  * block matches the transid in the parent node's pointer.  This is how we
322  * detect blocks that either didn't get written at all or got written
323  * in the wrong place.
324  */
325 static int verify_parent_transid(struct extent_io_tree *io_tree,
326                                  struct extent_buffer *eb, u64 parent_transid,
327                                  int atomic)
328 {
329         struct extent_state *cached_state = NULL;
330         int ret;
331         bool need_lock = (current->journal_info == BTRFS_SEND_TRANS_STUB);
332
333         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
334                 return 0;
335
336         if (atomic)
337                 return -EAGAIN;
338
339         if (need_lock) {
340                 btrfs_tree_read_lock(eb);
341                 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
342         }
343
344         lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1,
345                          &cached_state);
346         if (extent_buffer_uptodate(eb) &&
347             btrfs_header_generation(eb) == parent_transid) {
348                 ret = 0;
349                 goto out;
350         }
351         btrfs_err_rl(eb->fs_info,
352                 "parent transid verify failed on %llu wanted %llu found %llu",
353                         eb->start,
354                         parent_transid, btrfs_header_generation(eb));
355         ret = 1;
356
357         /*
358          * Things reading via commit roots that don't have normal protection,
359          * like send, can have a really old block in cache that may point at a
360          * block that has been freed and re-allocated.  So don't clear uptodate
361          * if we find an eb that is under IO (dirty/writeback) because we could
362          * end up reading in the stale data and then writing it back out and
363          * making everybody very sad.
364          */
365         if (!extent_buffer_under_io(eb))
366                 clear_extent_buffer_uptodate(eb);
367 out:
368         unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1,
369                              &cached_state);
370         if (need_lock)
371                 btrfs_tree_read_unlock_blocking(eb);
372         return ret;
373 }
374
375 /*
376  * Return 0 if the superblock checksum type matches the checksum value of that
377  * algorithm. Pass the raw disk superblock data.
378  */
379 static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
380                                   char *raw_disk_sb)
381 {
382         struct btrfs_super_block *disk_sb =
383                 (struct btrfs_super_block *)raw_disk_sb;
384         u16 csum_type = btrfs_super_csum_type(disk_sb);
385         int ret = 0;
386
387         if (csum_type == BTRFS_CSUM_TYPE_CRC32) {
388                 u32 crc = ~(u32)0;
389                 char result[sizeof(crc)];
390
391                 /*
392                  * The super_block structure does not span the whole
393                  * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space
394                  * is filled with zeros and is included in the checksum.
395                  */
396                 crc = btrfs_csum_data(raw_disk_sb + BTRFS_CSUM_SIZE,
397                                 crc, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
398                 btrfs_csum_final(crc, result);
399
400                 if (memcmp(raw_disk_sb, result, sizeof(result)))
401                         ret = 1;
402         }
403
404         if (csum_type >= ARRAY_SIZE(btrfs_csum_sizes)) {
405                 btrfs_err(fs_info, "unsupported checksum algorithm %u",
406                                 csum_type);
407                 ret = 1;
408         }
409
410         return ret;
411 }
412
413 static int verify_level_key(struct btrfs_fs_info *fs_info,
414                             struct extent_buffer *eb, int level,
415                             struct btrfs_key *first_key, u64 parent_transid)
416 {
417         int found_level;
418         struct btrfs_key found_key;
419         int ret;
420
421         found_level = btrfs_header_level(eb);
422         if (found_level != level) {
423 #ifdef CONFIG_BTRFS_DEBUG
424                 WARN_ON(1);
425                 btrfs_err(fs_info,
426 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
427                           eb->start, level, found_level);
428 #endif
429                 return -EIO;
430         }
431
432         if (!first_key)
433                 return 0;
434
435         /*
436          * For live tree block (new tree blocks in current transaction),
437          * we need proper lock context to avoid race, which is impossible here.
438          * So we only checks tree blocks which is read from disk, whose
439          * generation <= fs_info->last_trans_committed.
440          */
441         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
442                 return 0;
443         if (found_level)
444                 btrfs_node_key_to_cpu(eb, &found_key, 0);
445         else
446                 btrfs_item_key_to_cpu(eb, &found_key, 0);
447         ret = btrfs_comp_cpu_keys(first_key, &found_key);
448
449 #ifdef CONFIG_BTRFS_DEBUG
450         if (ret) {
451                 WARN_ON(1);
452                 btrfs_err(fs_info,
453 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
454                           eb->start, parent_transid, first_key->objectid,
455                           first_key->type, first_key->offset,
456                           found_key.objectid, found_key.type,
457                           found_key.offset);
458         }
459 #endif
460         return ret;
461 }
462
463 /*
464  * helper to read a given tree block, doing retries as required when
465  * the checksums don't match and we have alternate mirrors to try.
466  *
467  * @parent_transid:     expected transid, skip check if 0
468  * @level:              expected level, mandatory check
469  * @first_key:          expected key of first slot, skip check if NULL
470  */
471 static int btree_read_extent_buffer_pages(struct btrfs_fs_info *fs_info,
472                                           struct extent_buffer *eb,
473                                           u64 parent_transid, int level,
474                                           struct btrfs_key *first_key)
475 {
476         struct extent_io_tree *io_tree;
477         int failed = 0;
478         int ret;
479         int num_copies = 0;
480         int mirror_num = 0;
481         int failed_mirror = 0;
482
483         clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
484         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
485         while (1) {
486                 ret = read_extent_buffer_pages(io_tree, eb, WAIT_COMPLETE,
487                                                mirror_num);
488                 if (!ret) {
489                         if (verify_parent_transid(io_tree, eb,
490                                                    parent_transid, 0))
491                                 ret = -EIO;
492                         else if (verify_level_key(fs_info, eb, level,
493                                                   first_key, parent_transid))
494                                 ret = -EUCLEAN;
495                         else
496                                 break;
497                 }
498
499                 /*
500                  * This buffer's crc is fine, but its contents are corrupted, so
501                  * there is no reason to read the other copies, they won't be
502                  * any less wrong.
503                  */
504                 if (test_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags) ||
505                     ret == -EUCLEAN)
506                         break;
507
508                 num_copies = btrfs_num_copies(fs_info,
509                                               eb->start, eb->len);
510                 if (num_copies == 1)
511                         break;
512
513                 if (!failed_mirror) {
514                         failed = 1;
515                         failed_mirror = eb->read_mirror;
516                 }
517
518                 mirror_num++;
519                 if (mirror_num == failed_mirror)
520                         mirror_num++;
521
522                 if (mirror_num > num_copies)
523                         break;
524         }
525
526         if (failed && !ret && failed_mirror)
527                 repair_eb_io_failure(fs_info, eb, failed_mirror);
528
529         return ret;
530 }
531
532 /*
533  * checksum a dirty tree block before IO.  This has extra checks to make sure
534  * we only fill in the checksum field in the first page of a multi-page block
535  */
536
537 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct page *page)
538 {
539         u64 start = page_offset(page);
540         u64 found_start;
541         struct extent_buffer *eb;
542
543         eb = (struct extent_buffer *)page->private;
544         if (page != eb->pages[0])
545                 return 0;
546
547         found_start = btrfs_header_bytenr(eb);
548         /*
549          * Please do not consolidate these warnings into a single if.
550          * It is useful to know what went wrong.
551          */
552         if (WARN_ON(found_start != start))
553                 return -EUCLEAN;
554         if (WARN_ON(!PageUptodate(page)))
555                 return -EUCLEAN;
556
557         ASSERT(memcmp_extent_buffer(eb, fs_info->fsid,
558                         btrfs_header_fsid(), BTRFS_FSID_SIZE) == 0);
559
560         return csum_tree_block(fs_info, eb, 0);
561 }
562
563 static int check_tree_block_fsid(struct btrfs_fs_info *fs_info,
564                                  struct extent_buffer *eb)
565 {
566         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
567         u8 fsid[BTRFS_FSID_SIZE];
568         int ret = 1;
569
570         read_extent_buffer(eb, fsid, btrfs_header_fsid(), BTRFS_FSID_SIZE);
571         while (fs_devices) {
572                 if (!memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE)) {
573                         ret = 0;
574                         break;
575                 }
576                 fs_devices = fs_devices->seed;
577         }
578         return ret;
579 }
580
581 static int btree_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
582                                       u64 phy_offset, struct page *page,
583                                       u64 start, u64 end, int mirror)
584 {
585         u64 found_start;
586         int found_level;
587         struct extent_buffer *eb;
588         struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
589         struct btrfs_fs_info *fs_info = root->fs_info;
590         int ret = 0;
591         int reads_done;
592
593         if (!page->private)
594                 goto out;
595
596         eb = (struct extent_buffer *)page->private;
597
598         /* the pending IO might have been the only thing that kept this buffer
599          * in memory.  Make sure we have a ref for all this other checks
600          */
601         extent_buffer_get(eb);
602
603         reads_done = atomic_dec_and_test(&eb->io_pages);
604         if (!reads_done)
605                 goto err;
606
607         eb->read_mirror = mirror;
608         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
609                 ret = -EIO;
610                 goto err;
611         }
612
613         found_start = btrfs_header_bytenr(eb);
614         if (found_start != eb->start) {
615                 btrfs_err_rl(fs_info, "bad tree block start, want %llu have %llu",
616                              eb->start, found_start);
617                 ret = -EIO;
618                 goto err;
619         }
620         if (check_tree_block_fsid(fs_info, eb)) {
621                 btrfs_err_rl(fs_info, "bad fsid on block %llu",
622                              eb->start);
623                 ret = -EIO;
624                 goto err;
625         }
626         found_level = btrfs_header_level(eb);
627         if (found_level >= BTRFS_MAX_LEVEL) {
628                 btrfs_err(fs_info, "bad tree block level %d on %llu",
629                           (int)btrfs_header_level(eb), eb->start);
630                 ret = -EIO;
631                 goto err;
632         }
633
634         btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb),
635                                        eb, found_level);
636
637         ret = csum_tree_block(fs_info, eb, 1);
638         if (ret)
639                 goto err;
640
641         /*
642          * If this is a leaf block and it is corrupt, set the corrupt bit so
643          * that we don't try and read the other copies of this block, just
644          * return -EIO.
645          */
646         if (found_level == 0 && btrfs_check_leaf_full(fs_info, eb)) {
647                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
648                 ret = -EIO;
649         }
650
651         if (found_level > 0 && btrfs_check_node(fs_info, eb))
652                 ret = -EIO;
653
654         if (!ret)
655                 set_extent_buffer_uptodate(eb);
656 err:
657         if (reads_done &&
658             test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags))
659                 btree_readahead_hook(eb, ret);
660
661         if (ret) {
662                 /*
663                  * our io error hook is going to dec the io pages
664                  * again, we have to make sure it has something
665                  * to decrement
666                  */
667                 atomic_inc(&eb->io_pages);
668                 clear_extent_buffer_uptodate(eb);
669         }
670         free_extent_buffer(eb);
671 out:
672         return ret;
673 }
674
675 static int btree_io_failed_hook(struct page *page, int failed_mirror)
676 {
677         struct extent_buffer *eb;
678
679         eb = (struct extent_buffer *)page->private;
680         set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
681         eb->read_mirror = failed_mirror;
682         atomic_dec(&eb->io_pages);
683         if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags))
684                 btree_readahead_hook(eb, -EIO);
685         return -EIO;    /* we fixed nothing */
686 }
687
688 static void end_workqueue_bio(struct bio *bio)
689 {
690         struct btrfs_end_io_wq *end_io_wq = bio->bi_private;
691         struct btrfs_fs_info *fs_info;
692         struct btrfs_workqueue *wq;
693         btrfs_work_func_t func;
694
695         fs_info = end_io_wq->info;
696         end_io_wq->status = bio->bi_status;
697
698         if (bio_op(bio) == REQ_OP_WRITE) {
699                 if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) {
700                         wq = fs_info->endio_meta_write_workers;
701                         func = btrfs_endio_meta_write_helper;
702                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_FREE_SPACE) {
703                         wq = fs_info->endio_freespace_worker;
704                         func = btrfs_freespace_write_helper;
705                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) {
706                         wq = fs_info->endio_raid56_workers;
707                         func = btrfs_endio_raid56_helper;
708                 } else {
709                         wq = fs_info->endio_write_workers;
710                         func = btrfs_endio_write_helper;
711                 }
712         } else {
713                 if (unlikely(end_io_wq->metadata ==
714                              BTRFS_WQ_ENDIO_DIO_REPAIR)) {
715                         wq = fs_info->endio_repair_workers;
716                         func = btrfs_endio_repair_helper;
717                 } else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) {
718                         wq = fs_info->endio_raid56_workers;
719                         func = btrfs_endio_raid56_helper;
720                 } else if (end_io_wq->metadata) {
721                         wq = fs_info->endio_meta_workers;
722                         func = btrfs_endio_meta_helper;
723                 } else {
724                         wq = fs_info->endio_workers;
725                         func = btrfs_endio_helper;
726                 }
727         }
728
729         btrfs_init_work(&end_io_wq->work, func, end_workqueue_fn, NULL, NULL);
730         btrfs_queue_work(wq, &end_io_wq->work);
731 }
732
733 blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
734                         enum btrfs_wq_endio_type metadata)
735 {
736         struct btrfs_end_io_wq *end_io_wq;
737
738         end_io_wq = kmem_cache_alloc(btrfs_end_io_wq_cache, GFP_NOFS);
739         if (!end_io_wq)
740                 return BLK_STS_RESOURCE;
741
742         end_io_wq->private = bio->bi_private;
743         end_io_wq->end_io = bio->bi_end_io;
744         end_io_wq->info = info;
745         end_io_wq->status = 0;
746         end_io_wq->bio = bio;
747         end_io_wq->metadata = metadata;
748
749         bio->bi_private = end_io_wq;
750         bio->bi_end_io = end_workqueue_bio;
751         return 0;
752 }
753
754 static void run_one_async_start(struct btrfs_work *work)
755 {
756         struct async_submit_bio *async;
757         blk_status_t ret;
758
759         async = container_of(work, struct  async_submit_bio, work);
760         ret = async->submit_bio_start(async->private_data, async->bio,
761                                       async->bio_offset);
762         if (ret)
763                 async->status = ret;
764 }
765
766 static void run_one_async_done(struct btrfs_work *work)
767 {
768         struct async_submit_bio *async;
769
770         async = container_of(work, struct  async_submit_bio, work);
771
772         /* If an error occurred we just want to clean up the bio and move on */
773         if (async->status) {
774                 async->bio->bi_status = async->status;
775                 bio_endio(async->bio);
776                 return;
777         }
778
779         async->submit_bio_done(async->private_data, async->bio, async->mirror_num);
780 }
781
782 static void run_one_async_free(struct btrfs_work *work)
783 {
784         struct async_submit_bio *async;
785
786         async = container_of(work, struct  async_submit_bio, work);
787         kfree(async);
788 }
789
790 blk_status_t btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
791                                  int mirror_num, unsigned long bio_flags,
792                                  u64 bio_offset, void *private_data,
793                                  extent_submit_bio_start_t *submit_bio_start,
794                                  extent_submit_bio_done_t *submit_bio_done)
795 {
796         struct async_submit_bio *async;
797
798         async = kmalloc(sizeof(*async), GFP_NOFS);
799         if (!async)
800                 return BLK_STS_RESOURCE;
801
802         async->private_data = private_data;
803         async->bio = bio;
804         async->mirror_num = mirror_num;
805         async->submit_bio_start = submit_bio_start;
806         async->submit_bio_done = submit_bio_done;
807
808         btrfs_init_work(&async->work, btrfs_worker_helper, run_one_async_start,
809                         run_one_async_done, run_one_async_free);
810
811         async->bio_flags = bio_flags;
812         async->bio_offset = bio_offset;
813
814         async->status = 0;
815
816         if (op_is_sync(bio->bi_opf))
817                 btrfs_set_work_high_priority(&async->work);
818
819         btrfs_queue_work(fs_info->workers, &async->work);
820         return 0;
821 }
822
823 static blk_status_t btree_csum_one_bio(struct bio *bio)
824 {
825         struct bio_vec *bvec;
826         struct btrfs_root *root;
827         int i, ret = 0;
828
829         ASSERT(!bio_flagged(bio, BIO_CLONED));
830         bio_for_each_segment_all(bvec, bio, i) {
831                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
832                 ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
833                 if (ret)
834                         break;
835         }
836
837         return errno_to_blk_status(ret);
838 }
839
840 static blk_status_t btree_submit_bio_start(void *private_data, struct bio *bio,
841                                              u64 bio_offset)
842 {
843         /*
844          * when we're called for a write, we're already in the async
845          * submission context.  Just jump into btrfs_map_bio
846          */
847         return btree_csum_one_bio(bio);
848 }
849
850 static blk_status_t btree_submit_bio_done(void *private_data, struct bio *bio,
851                                             int mirror_num)
852 {
853         struct inode *inode = private_data;
854         blk_status_t ret;
855
856         /*
857          * when we're called for a write, we're already in the async
858          * submission context.  Just jump into btrfs_map_bio
859          */
860         ret = btrfs_map_bio(btrfs_sb(inode->i_sb), bio, mirror_num, 1);
861         if (ret) {
862                 bio->bi_status = ret;
863                 bio_endio(bio);
864         }
865         return ret;
866 }
867
868 static int check_async_write(struct btrfs_inode *bi)
869 {
870         if (atomic_read(&bi->sync_writers))
871                 return 0;
872 #ifdef CONFIG_X86
873         if (static_cpu_has(X86_FEATURE_XMM4_2))
874                 return 0;
875 #endif
876         return 1;
877 }
878
879 static blk_status_t btree_submit_bio_hook(void *private_data, struct bio *bio,
880                                           int mirror_num, unsigned long bio_flags,
881                                           u64 bio_offset)
882 {
883         struct inode *inode = private_data;
884         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
885         int async = check_async_write(BTRFS_I(inode));
886         blk_status_t ret;
887
888         if (bio_op(bio) != REQ_OP_WRITE) {
889                 /*
890                  * called for a read, do the setup so that checksum validation
891                  * can happen in the async kernel threads
892                  */
893                 ret = btrfs_bio_wq_end_io(fs_info, bio,
894                                           BTRFS_WQ_ENDIO_METADATA);
895                 if (ret)
896                         goto out_w_error;
897                 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
898         } else if (!async) {
899                 ret = btree_csum_one_bio(bio);
900                 if (ret)
901                         goto out_w_error;
902                 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
903         } else {
904                 /*
905                  * kthread helpers are used to submit writes so that
906                  * checksumming can happen in parallel across all CPUs
907                  */
908                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, 0,
909                                           bio_offset, private_data,
910                                           btree_submit_bio_start,
911                                           btree_submit_bio_done);
912         }
913
914         if (ret)
915                 goto out_w_error;
916         return 0;
917
918 out_w_error:
919         bio->bi_status = ret;
920         bio_endio(bio);
921         return ret;
922 }
923
924 #ifdef CONFIG_MIGRATION
925 static int btree_migratepage(struct address_space *mapping,
926                         struct page *newpage, struct page *page,
927                         enum migrate_mode mode)
928 {
929         /*
930          * we can't safely write a btree page from here,
931          * we haven't done the locking hook
932          */
933         if (PageDirty(page))
934                 return -EAGAIN;
935         /*
936          * Buffers may be managed in a filesystem specific way.
937          * We must have no buffers or drop them.
938          */
939         if (page_has_private(page) &&
940             !try_to_release_page(page, GFP_KERNEL))
941                 return -EAGAIN;
942         return migrate_page(mapping, newpage, page, mode);
943 }
944 #endif
945
946
947 static int btree_writepages(struct address_space *mapping,
948                             struct writeback_control *wbc)
949 {
950         struct btrfs_fs_info *fs_info;
951         int ret;
952
953         if (wbc->sync_mode == WB_SYNC_NONE) {
954
955                 if (wbc->for_kupdate)
956                         return 0;
957
958                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
959                 /* this is a bit racy, but that's ok */
960                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
961                                              BTRFS_DIRTY_METADATA_THRESH,
962                                              fs_info->dirty_metadata_batch);
963                 if (ret < 0)
964                         return 0;
965         }
966         return btree_write_cache_pages(mapping, wbc);
967 }
968
969 static int btree_readpage(struct file *file, struct page *page)
970 {
971         struct extent_io_tree *tree;
972         tree = &BTRFS_I(page->mapping->host)->io_tree;
973         return extent_read_full_page(tree, page, btree_get_extent, 0);
974 }
975
976 static int btree_releasepage(struct page *page, gfp_t gfp_flags)
977 {
978         if (PageWriteback(page) || PageDirty(page))
979                 return 0;
980
981         return try_release_extent_buffer(page);
982 }
983
984 static void btree_invalidatepage(struct page *page, unsigned int offset,
985                                  unsigned int length)
986 {
987         struct extent_io_tree *tree;
988         tree = &BTRFS_I(page->mapping->host)->io_tree;
989         extent_invalidatepage(tree, page, offset);
990         btree_releasepage(page, GFP_NOFS);
991         if (PagePrivate(page)) {
992                 btrfs_warn(BTRFS_I(page->mapping->host)->root->fs_info,
993                            "page private not zero on page %llu",
994                            (unsigned long long)page_offset(page));
995                 ClearPagePrivate(page);
996                 set_page_private(page, 0);
997                 put_page(page);
998         }
999 }
1000
1001 static int btree_set_page_dirty(struct page *page)
1002 {
1003 #ifdef DEBUG
1004         struct extent_buffer *eb;
1005
1006         BUG_ON(!PagePrivate(page));
1007         eb = (struct extent_buffer *)page->private;
1008         BUG_ON(!eb);
1009         BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
1010         BUG_ON(!atomic_read(&eb->refs));
1011         btrfs_assert_tree_locked(eb);
1012 #endif
1013         return __set_page_dirty_nobuffers(page);
1014 }
1015
1016 static const struct address_space_operations btree_aops = {
1017         .readpage       = btree_readpage,
1018         .writepages     = btree_writepages,
1019         .releasepage    = btree_releasepage,
1020         .invalidatepage = btree_invalidatepage,
1021 #ifdef CONFIG_MIGRATION
1022         .migratepage    = btree_migratepage,
1023 #endif
1024         .set_page_dirty = btree_set_page_dirty,
1025 };
1026
1027 void readahead_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr)
1028 {
1029         struct extent_buffer *buf = NULL;
1030         struct inode *btree_inode = fs_info->btree_inode;
1031
1032         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1033         if (IS_ERR(buf))
1034                 return;
1035         read_extent_buffer_pages(&BTRFS_I(btree_inode)->io_tree,
1036                                  buf, WAIT_NONE, 0);
1037         free_extent_buffer(buf);
1038 }
1039
1040 int reada_tree_block_flagged(struct btrfs_fs_info *fs_info, u64 bytenr,
1041                          int mirror_num, struct extent_buffer **eb)
1042 {
1043         struct extent_buffer *buf = NULL;
1044         struct inode *btree_inode = fs_info->btree_inode;
1045         struct extent_io_tree *io_tree = &BTRFS_I(btree_inode)->io_tree;
1046         int ret;
1047
1048         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1049         if (IS_ERR(buf))
1050                 return 0;
1051
1052         set_bit(EXTENT_BUFFER_READAHEAD, &buf->bflags);
1053
1054         ret = read_extent_buffer_pages(io_tree, buf, WAIT_PAGE_LOCK,
1055                                        mirror_num);
1056         if (ret) {
1057                 free_extent_buffer(buf);
1058                 return ret;
1059         }
1060
1061         if (test_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags)) {
1062                 free_extent_buffer(buf);
1063                 return -EIO;
1064         } else if (extent_buffer_uptodate(buf)) {
1065                 *eb = buf;
1066         } else {
1067                 free_extent_buffer(buf);
1068         }
1069         return 0;
1070 }
1071
1072 struct extent_buffer *btrfs_find_create_tree_block(
1073                                                 struct btrfs_fs_info *fs_info,
1074                                                 u64 bytenr)
1075 {
1076         if (btrfs_is_testing(fs_info))
1077                 return alloc_test_extent_buffer(fs_info, bytenr);
1078         return alloc_extent_buffer(fs_info, bytenr);
1079 }
1080
1081
1082 int btrfs_write_tree_block(struct extent_buffer *buf)
1083 {
1084         return filemap_fdatawrite_range(buf->pages[0]->mapping, buf->start,
1085                                         buf->start + buf->len - 1);
1086 }
1087
1088 void btrfs_wait_tree_block_writeback(struct extent_buffer *buf)
1089 {
1090         filemap_fdatawait_range(buf->pages[0]->mapping,
1091                                 buf->start, buf->start + buf->len - 1);
1092 }
1093
1094 /*
1095  * Read tree block at logical address @bytenr and do variant basic but critical
1096  * verification.
1097  *
1098  * @parent_transid:     expected transid of this tree block, skip check if 0
1099  * @level:              expected level, mandatory check
1100  * @first_key:          expected key in slot 0, skip check if NULL
1101  */
1102 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
1103                                       u64 parent_transid, int level,
1104                                       struct btrfs_key *first_key)
1105 {
1106         struct extent_buffer *buf = NULL;
1107         int ret;
1108
1109         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1110         if (IS_ERR(buf))
1111                 return buf;
1112
1113         ret = btree_read_extent_buffer_pages(fs_info, buf, parent_transid,
1114                                              level, first_key);
1115         if (ret) {
1116                 free_extent_buffer(buf);
1117                 return ERR_PTR(ret);
1118         }
1119         return buf;
1120
1121 }
1122
1123 void clean_tree_block(struct btrfs_fs_info *fs_info,
1124                       struct extent_buffer *buf)
1125 {
1126         if (btrfs_header_generation(buf) ==
1127             fs_info->running_transaction->transid) {
1128                 btrfs_assert_tree_locked(buf);
1129
1130                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
1131                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1132                                                  -buf->len,
1133                                                  fs_info->dirty_metadata_batch);
1134                         /* ugh, clear_extent_buffer_dirty needs to lock the page */
1135                         btrfs_set_lock_blocking(buf);
1136                         clear_extent_buffer_dirty(buf);
1137                 }
1138         }
1139 }
1140
1141 static struct btrfs_subvolume_writers *btrfs_alloc_subvolume_writers(void)
1142 {
1143         struct btrfs_subvolume_writers *writers;
1144         int ret;
1145
1146         writers = kmalloc(sizeof(*writers), GFP_NOFS);
1147         if (!writers)
1148                 return ERR_PTR(-ENOMEM);
1149
1150         ret = percpu_counter_init(&writers->counter, 0, GFP_NOFS);
1151         if (ret < 0) {
1152                 kfree(writers);
1153                 return ERR_PTR(ret);
1154         }
1155
1156         init_waitqueue_head(&writers->wait);
1157         return writers;
1158 }
1159
1160 static void
1161 btrfs_free_subvolume_writers(struct btrfs_subvolume_writers *writers)
1162 {
1163         percpu_counter_destroy(&writers->counter);
1164         kfree(writers);
1165 }
1166
1167 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
1168                          u64 objectid)
1169 {
1170         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
1171         root->node = NULL;
1172         root->commit_root = NULL;
1173         root->state = 0;
1174         root->orphan_cleanup_state = 0;
1175
1176         root->objectid = objectid;
1177         root->last_trans = 0;
1178         root->highest_objectid = 0;
1179         root->nr_delalloc_inodes = 0;
1180         root->nr_ordered_extents = 0;
1181         root->name = NULL;
1182         root->inode_tree = RB_ROOT;
1183         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
1184         root->block_rsv = NULL;
1185
1186         INIT_LIST_HEAD(&root->dirty_list);
1187         INIT_LIST_HEAD(&root->root_list);
1188         INIT_LIST_HEAD(&root->delalloc_inodes);
1189         INIT_LIST_HEAD(&root->delalloc_root);
1190         INIT_LIST_HEAD(&root->ordered_extents);
1191         INIT_LIST_HEAD(&root->ordered_root);
1192         INIT_LIST_HEAD(&root->logged_list[0]);
1193         INIT_LIST_HEAD(&root->logged_list[1]);
1194         spin_lock_init(&root->inode_lock);
1195         spin_lock_init(&root->delalloc_lock);
1196         spin_lock_init(&root->ordered_extent_lock);
1197         spin_lock_init(&root->accounting_lock);
1198         spin_lock_init(&root->log_extents_lock[0]);
1199         spin_lock_init(&root->log_extents_lock[1]);
1200         spin_lock_init(&root->qgroup_meta_rsv_lock);
1201         mutex_init(&root->objectid_mutex);
1202         mutex_init(&root->log_mutex);
1203         mutex_init(&root->ordered_extent_mutex);
1204         mutex_init(&root->delalloc_mutex);
1205         init_waitqueue_head(&root->log_writer_wait);
1206         init_waitqueue_head(&root->log_commit_wait[0]);
1207         init_waitqueue_head(&root->log_commit_wait[1]);
1208         INIT_LIST_HEAD(&root->log_ctxs[0]);
1209         INIT_LIST_HEAD(&root->log_ctxs[1]);
1210         atomic_set(&root->log_commit[0], 0);
1211         atomic_set(&root->log_commit[1], 0);
1212         atomic_set(&root->log_writers, 0);
1213         atomic_set(&root->log_batch, 0);
1214         refcount_set(&root->refs, 1);
1215         atomic_set(&root->will_be_snapshotted, 0);
1216         root->log_transid = 0;
1217         root->log_transid_committed = -1;
1218         root->last_log_commit = 0;
1219         if (!dummy)
1220                 extent_io_tree_init(&root->dirty_log_pages, NULL);
1221
1222         memset(&root->root_key, 0, sizeof(root->root_key));
1223         memset(&root->root_item, 0, sizeof(root->root_item));
1224         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
1225         if (!dummy)
1226                 root->defrag_trans_start = fs_info->generation;
1227         else
1228                 root->defrag_trans_start = 0;
1229         root->root_key.objectid = objectid;
1230         root->anon_dev = 0;
1231
1232         spin_lock_init(&root->root_item_lock);
1233 }
1234
1235 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1236                 gfp_t flags)
1237 {
1238         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1239         if (root)
1240                 root->fs_info = fs_info;
1241         return root;
1242 }
1243
1244 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1245 /* Should only be used by the testing infrastructure */
1246 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1247 {
1248         struct btrfs_root *root;
1249
1250         if (!fs_info)
1251                 return ERR_PTR(-EINVAL);
1252
1253         root = btrfs_alloc_root(fs_info, GFP_KERNEL);
1254         if (!root)
1255                 return ERR_PTR(-ENOMEM);
1256
1257         /* We don't use the stripesize in selftest, set it as sectorsize */
1258         __setup_root(root, fs_info, BTRFS_ROOT_TREE_OBJECTID);
1259         root->alloc_bytenr = 0;
1260
1261         return root;
1262 }
1263 #endif
1264
1265 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1266                                      struct btrfs_fs_info *fs_info,
1267                                      u64 objectid)
1268 {
1269         struct extent_buffer *leaf;
1270         struct btrfs_root *tree_root = fs_info->tree_root;
1271         struct btrfs_root *root;
1272         struct btrfs_key key;
1273         int ret = 0;
1274         uuid_le uuid = NULL_UUID_LE;
1275
1276         root = btrfs_alloc_root(fs_info, GFP_KERNEL);
1277         if (!root)
1278                 return ERR_PTR(-ENOMEM);
1279
1280         __setup_root(root, fs_info, objectid);
1281         root->root_key.objectid = objectid;
1282         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1283         root->root_key.offset = 0;
1284
1285         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0);
1286         if (IS_ERR(leaf)) {
1287                 ret = PTR_ERR(leaf);
1288                 leaf = NULL;
1289                 goto fail;
1290         }
1291
1292         root->node = leaf;
1293         btrfs_mark_buffer_dirty(leaf);
1294
1295         root->commit_root = btrfs_root_node(root);
1296         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1297
1298         root->root_item.flags = 0;
1299         root->root_item.byte_limit = 0;
1300         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1301         btrfs_set_root_generation(&root->root_item, trans->transid);
1302         btrfs_set_root_level(&root->root_item, 0);
1303         btrfs_set_root_refs(&root->root_item, 1);
1304         btrfs_set_root_used(&root->root_item, leaf->len);
1305         btrfs_set_root_last_snapshot(&root->root_item, 0);
1306         btrfs_set_root_dirid(&root->root_item, 0);
1307         if (is_fstree(objectid))
1308                 uuid_le_gen(&uuid);
1309         memcpy(root->root_item.uuid, uuid.b, BTRFS_UUID_SIZE);
1310         root->root_item.drop_level = 0;
1311
1312         key.objectid = objectid;
1313         key.type = BTRFS_ROOT_ITEM_KEY;
1314         key.offset = 0;
1315         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1316         if (ret)
1317                 goto fail;
1318
1319         btrfs_tree_unlock(leaf);
1320
1321         return root;
1322
1323 fail:
1324         if (leaf) {
1325                 btrfs_tree_unlock(leaf);
1326                 free_extent_buffer(root->commit_root);
1327                 free_extent_buffer(leaf);
1328         }
1329         kfree(root);
1330
1331         return ERR_PTR(ret);
1332 }
1333
1334 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1335                                          struct btrfs_fs_info *fs_info)
1336 {
1337         struct btrfs_root *root;
1338         struct extent_buffer *leaf;
1339
1340         root = btrfs_alloc_root(fs_info, GFP_NOFS);
1341         if (!root)
1342                 return ERR_PTR(-ENOMEM);
1343
1344         __setup_root(root, fs_info, BTRFS_TREE_LOG_OBJECTID);
1345
1346         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1347         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1348         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1349
1350         /*
1351          * DON'T set REF_COWS for log trees
1352          *
1353          * log trees do not get reference counted because they go away
1354          * before a real commit is actually done.  They do store pointers
1355          * to file data extents, and those reference counts still get
1356          * updated (along with back refs to the log tree).
1357          */
1358
1359         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1360                         NULL, 0, 0, 0);
1361         if (IS_ERR(leaf)) {
1362                 kfree(root);
1363                 return ERR_CAST(leaf);
1364         }
1365
1366         root->node = leaf;
1367
1368         btrfs_mark_buffer_dirty(root->node);
1369         btrfs_tree_unlock(root->node);
1370         return root;
1371 }
1372
1373 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1374                              struct btrfs_fs_info *fs_info)
1375 {
1376         struct btrfs_root *log_root;
1377
1378         log_root = alloc_log_tree(trans, fs_info);
1379         if (IS_ERR(log_root))
1380                 return PTR_ERR(log_root);
1381         WARN_ON(fs_info->log_root_tree);
1382         fs_info->log_root_tree = log_root;
1383         return 0;
1384 }
1385
1386 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1387                        struct btrfs_root *root)
1388 {
1389         struct btrfs_fs_info *fs_info = root->fs_info;
1390         struct btrfs_root *log_root;
1391         struct btrfs_inode_item *inode_item;
1392
1393         log_root = alloc_log_tree(trans, fs_info);
1394         if (IS_ERR(log_root))
1395                 return PTR_ERR(log_root);
1396
1397         log_root->last_trans = trans->transid;
1398         log_root->root_key.offset = root->root_key.objectid;
1399
1400         inode_item = &log_root->root_item.inode;
1401         btrfs_set_stack_inode_generation(inode_item, 1);
1402         btrfs_set_stack_inode_size(inode_item, 3);
1403         btrfs_set_stack_inode_nlink(inode_item, 1);
1404         btrfs_set_stack_inode_nbytes(inode_item,
1405                                      fs_info->nodesize);
1406         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1407
1408         btrfs_set_root_node(&log_root->root_item, log_root->node);
1409
1410         WARN_ON(root->log_root);
1411         root->log_root = log_root;
1412         root->log_transid = 0;
1413         root->log_transid_committed = -1;
1414         root->last_log_commit = 0;
1415         return 0;
1416 }
1417
1418 static struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1419                                                struct btrfs_key *key)
1420 {
1421         struct btrfs_root *root;
1422         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1423         struct btrfs_path *path;
1424         u64 generation;
1425         int ret;
1426         int level;
1427
1428         path = btrfs_alloc_path();
1429         if (!path)
1430                 return ERR_PTR(-ENOMEM);
1431
1432         root = btrfs_alloc_root(fs_info, GFP_NOFS);
1433         if (!root) {
1434                 ret = -ENOMEM;
1435                 goto alloc_fail;
1436         }
1437
1438         __setup_root(root, fs_info, key->objectid);
1439
1440         ret = btrfs_find_root(tree_root, key, path,
1441                               &root->root_item, &root->root_key);
1442         if (ret) {
1443                 if (ret > 0)
1444                         ret = -ENOENT;
1445                 goto find_fail;
1446         }
1447
1448         generation = btrfs_root_generation(&root->root_item);
1449         level = btrfs_root_level(&root->root_item);
1450         root->node = read_tree_block(fs_info,
1451                                      btrfs_root_bytenr(&root->root_item),
1452                                      generation, level, NULL);
1453         if (IS_ERR(root->node)) {
1454                 ret = PTR_ERR(root->node);
1455                 goto find_fail;
1456         } else if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1457                 ret = -EIO;
1458                 free_extent_buffer(root->node);
1459                 goto find_fail;
1460         }
1461         root->commit_root = btrfs_root_node(root);
1462 out:
1463         btrfs_free_path(path);
1464         return root;
1465
1466 find_fail:
1467         kfree(root);
1468 alloc_fail:
1469         root = ERR_PTR(ret);
1470         goto out;
1471 }
1472
1473 struct btrfs_root *btrfs_read_fs_root(struct btrfs_root *tree_root,
1474                                       struct btrfs_key *location)
1475 {
1476         struct btrfs_root *root;
1477
1478         root = btrfs_read_tree_root(tree_root, location);
1479         if (IS_ERR(root))
1480                 return root;
1481
1482         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
1483                 set_bit(BTRFS_ROOT_REF_COWS, &root->state);
1484                 btrfs_check_and_init_root_item(&root->root_item);
1485         }
1486
1487         return root;
1488 }
1489
1490 int btrfs_init_fs_root(struct btrfs_root *root)
1491 {
1492         int ret;
1493         struct btrfs_subvolume_writers *writers;
1494
1495         root->free_ino_ctl = kzalloc(sizeof(*root->free_ino_ctl), GFP_NOFS);
1496         root->free_ino_pinned = kzalloc(sizeof(*root->free_ino_pinned),
1497                                         GFP_NOFS);
1498         if (!root->free_ino_pinned || !root->free_ino_ctl) {
1499                 ret = -ENOMEM;
1500                 goto fail;
1501         }
1502
1503         writers = btrfs_alloc_subvolume_writers();
1504         if (IS_ERR(writers)) {
1505                 ret = PTR_ERR(writers);
1506                 goto fail;
1507         }
1508         root->subv_writers = writers;
1509
1510         btrfs_init_free_ino_ctl(root);
1511         spin_lock_init(&root->ino_cache_lock);
1512         init_waitqueue_head(&root->ino_cache_wait);
1513
1514         ret = get_anon_bdev(&root->anon_dev);
1515         if (ret)
1516                 goto fail;
1517
1518         mutex_lock(&root->objectid_mutex);
1519         ret = btrfs_find_highest_objectid(root,
1520                                         &root->highest_objectid);
1521         if (ret) {
1522                 mutex_unlock(&root->objectid_mutex);
1523                 goto fail;
1524         }
1525
1526         ASSERT(root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
1527
1528         mutex_unlock(&root->objectid_mutex);
1529
1530         return 0;
1531 fail:
1532         /* the caller is responsible to call free_fs_root */
1533         return ret;
1534 }
1535
1536 struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1537                                         u64 root_id)
1538 {
1539         struct btrfs_root *root;
1540
1541         spin_lock(&fs_info->fs_roots_radix_lock);
1542         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1543                                  (unsigned long)root_id);
1544         spin_unlock(&fs_info->fs_roots_radix_lock);
1545         return root;
1546 }
1547
1548 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1549                          struct btrfs_root *root)
1550 {
1551         int ret;
1552
1553         ret = radix_tree_preload(GFP_NOFS);
1554         if (ret)
1555                 return ret;
1556
1557         spin_lock(&fs_info->fs_roots_radix_lock);
1558         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1559                                 (unsigned long)root->root_key.objectid,
1560                                 root);
1561         if (ret == 0)
1562                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1563         spin_unlock(&fs_info->fs_roots_radix_lock);
1564         radix_tree_preload_end();
1565
1566         return ret;
1567 }
1568
1569 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1570                                      struct btrfs_key *location,
1571                                      bool check_ref)
1572 {
1573         struct btrfs_root *root;
1574         struct btrfs_path *path;
1575         struct btrfs_key key;
1576         int ret;
1577
1578         if (location->objectid == BTRFS_ROOT_TREE_OBJECTID)
1579                 return fs_info->tree_root;
1580         if (location->objectid == BTRFS_EXTENT_TREE_OBJECTID)
1581                 return fs_info->extent_root;
1582         if (location->objectid == BTRFS_CHUNK_TREE_OBJECTID)
1583                 return fs_info->chunk_root;
1584         if (location->objectid == BTRFS_DEV_TREE_OBJECTID)
1585                 return fs_info->dev_root;
1586         if (location->objectid == BTRFS_CSUM_TREE_OBJECTID)
1587                 return fs_info->csum_root;
1588         if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID)
1589                 return fs_info->quota_root ? fs_info->quota_root :
1590                                              ERR_PTR(-ENOENT);
1591         if (location->objectid == BTRFS_UUID_TREE_OBJECTID)
1592                 return fs_info->uuid_root ? fs_info->uuid_root :
1593                                             ERR_PTR(-ENOENT);
1594         if (location->objectid == BTRFS_FREE_SPACE_TREE_OBJECTID)
1595                 return fs_info->free_space_root ? fs_info->free_space_root :
1596                                                   ERR_PTR(-ENOENT);
1597 again:
1598         root = btrfs_lookup_fs_root(fs_info, location->objectid);
1599         if (root) {
1600                 if (check_ref && btrfs_root_refs(&root->root_item) == 0)
1601                         return ERR_PTR(-ENOENT);
1602                 return root;
1603         }
1604
1605         root = btrfs_read_fs_root(fs_info->tree_root, location);
1606         if (IS_ERR(root))
1607                 return root;
1608
1609         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1610                 ret = -ENOENT;
1611                 goto fail;
1612         }
1613
1614         ret = btrfs_init_fs_root(root);
1615         if (ret)
1616                 goto fail;
1617
1618         path = btrfs_alloc_path();
1619         if (!path) {
1620                 ret = -ENOMEM;
1621                 goto fail;
1622         }
1623         key.objectid = BTRFS_ORPHAN_OBJECTID;
1624         key.type = BTRFS_ORPHAN_ITEM_KEY;
1625         key.offset = location->objectid;
1626
1627         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1628         btrfs_free_path(path);
1629         if (ret < 0)
1630                 goto fail;
1631         if (ret == 0)
1632                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1633
1634         ret = btrfs_insert_fs_root(fs_info, root);
1635         if (ret) {
1636                 if (ret == -EEXIST) {
1637                         free_fs_root(root);
1638                         goto again;
1639                 }
1640                 goto fail;
1641         }
1642         return root;
1643 fail:
1644         free_fs_root(root);
1645         return ERR_PTR(ret);
1646 }
1647
1648 static int btrfs_congested_fn(void *congested_data, int bdi_bits)
1649 {
1650         struct btrfs_fs_info *info = (struct btrfs_fs_info *)congested_data;
1651         int ret = 0;
1652         struct btrfs_device *device;
1653         struct backing_dev_info *bdi;
1654
1655         rcu_read_lock();
1656         list_for_each_entry_rcu(device, &info->fs_devices->devices, dev_list) {
1657                 if (!device->bdev)
1658                         continue;
1659                 bdi = device->bdev->bd_bdi;
1660                 if (bdi_congested(bdi, bdi_bits)) {
1661                         ret = 1;
1662                         break;
1663                 }
1664         }
1665         rcu_read_unlock();
1666         return ret;
1667 }
1668
1669 /*
1670  * called by the kthread helper functions to finally call the bio end_io
1671  * functions.  This is where read checksum verification actually happens
1672  */
1673 static void end_workqueue_fn(struct btrfs_work *work)
1674 {
1675         struct bio *bio;
1676         struct btrfs_end_io_wq *end_io_wq;
1677
1678         end_io_wq = container_of(work, struct btrfs_end_io_wq, work);
1679         bio = end_io_wq->bio;
1680
1681         bio->bi_status = end_io_wq->status;
1682         bio->bi_private = end_io_wq->private;
1683         bio->bi_end_io = end_io_wq->end_io;
1684         kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
1685         bio_endio(bio);
1686 }
1687
1688 static int cleaner_kthread(void *arg)
1689 {
1690         struct btrfs_root *root = arg;
1691         struct btrfs_fs_info *fs_info = root->fs_info;
1692         int again;
1693         struct btrfs_trans_handle *trans;
1694
1695         do {
1696                 again = 0;
1697
1698                 /* Make the cleaner go to sleep early. */
1699                 if (btrfs_need_cleaner_sleep(fs_info))
1700                         goto sleep;
1701
1702                 /*
1703                  * Do not do anything if we might cause open_ctree() to block
1704                  * before we have finished mounting the filesystem.
1705                  */
1706                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1707                         goto sleep;
1708
1709                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1710                         goto sleep;
1711
1712                 /*
1713                  * Avoid the problem that we change the status of the fs
1714                  * during the above check and trylock.
1715                  */
1716                 if (btrfs_need_cleaner_sleep(fs_info)) {
1717                         mutex_unlock(&fs_info->cleaner_mutex);
1718                         goto sleep;
1719                 }
1720
1721                 mutex_lock(&fs_info->cleaner_delayed_iput_mutex);
1722                 btrfs_run_delayed_iputs(fs_info);
1723                 mutex_unlock(&fs_info->cleaner_delayed_iput_mutex);
1724
1725                 again = btrfs_clean_one_deleted_snapshot(root);
1726                 mutex_unlock(&fs_info->cleaner_mutex);
1727
1728                 /*
1729                  * The defragger has dealt with the R/O remount and umount,
1730                  * needn't do anything special here.
1731                  */
1732                 btrfs_run_defrag_inodes(fs_info);
1733
1734                 /*
1735                  * Acquires fs_info->delete_unused_bgs_mutex to avoid racing
1736                  * with relocation (btrfs_relocate_chunk) and relocation
1737                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1738                  * after acquiring fs_info->delete_unused_bgs_mutex. So we
1739                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1740                  * unused block groups.
1741                  */
1742                 btrfs_delete_unused_bgs(fs_info);
1743 sleep:
1744                 if (!again) {
1745                         set_current_state(TASK_INTERRUPTIBLE);
1746                         if (!kthread_should_stop())
1747                                 schedule();
1748                         __set_current_state(TASK_RUNNING);
1749                 }
1750         } while (!kthread_should_stop());
1751
1752         /*
1753          * Transaction kthread is stopped before us and wakes us up.
1754          * However we might have started a new transaction and COWed some
1755          * tree blocks when deleting unused block groups for example. So
1756          * make sure we commit the transaction we started to have a clean
1757          * shutdown when evicting the btree inode - if it has dirty pages
1758          * when we do the final iput() on it, eviction will trigger a
1759          * writeback for it which will fail with null pointer dereferences
1760          * since work queues and other resources were already released and
1761          * destroyed by the time the iput/eviction/writeback is made.
1762          */
1763         trans = btrfs_attach_transaction(root);
1764         if (IS_ERR(trans)) {
1765                 if (PTR_ERR(trans) != -ENOENT)
1766                         btrfs_err(fs_info,
1767                                   "cleaner transaction attach returned %ld",
1768                                   PTR_ERR(trans));
1769         } else {
1770                 int ret;
1771
1772                 ret = btrfs_commit_transaction(trans);
1773                 if (ret)
1774                         btrfs_err(fs_info,
1775                                   "cleaner open transaction commit returned %d",
1776                                   ret);
1777         }
1778
1779         return 0;
1780 }
1781
1782 static int transaction_kthread(void *arg)
1783 {
1784         struct btrfs_root *root = arg;
1785         struct btrfs_fs_info *fs_info = root->fs_info;
1786         struct btrfs_trans_handle *trans;
1787         struct btrfs_transaction *cur;
1788         u64 transid;
1789         time64_t now;
1790         unsigned long delay;
1791         bool cannot_commit;
1792
1793         do {
1794                 cannot_commit = false;
1795                 delay = HZ * fs_info->commit_interval;
1796                 mutex_lock(&fs_info->transaction_kthread_mutex);
1797
1798                 spin_lock(&fs_info->trans_lock);
1799                 cur = fs_info->running_transaction;
1800                 if (!cur) {
1801                         spin_unlock(&fs_info->trans_lock);
1802                         goto sleep;
1803                 }
1804
1805                 now = ktime_get_seconds();
1806                 if (cur->state < TRANS_STATE_BLOCKED &&
1807                     !test_bit(BTRFS_FS_NEED_ASYNC_COMMIT, &fs_info->flags) &&
1808                     (now < cur->start_time ||
1809                      now - cur->start_time < fs_info->commit_interval)) {
1810                         spin_unlock(&fs_info->trans_lock);
1811                         delay = HZ * 5;
1812                         goto sleep;
1813                 }
1814                 transid = cur->transid;
1815                 spin_unlock(&fs_info->trans_lock);
1816
1817                 /* If the file system is aborted, this will always fail. */
1818                 trans = btrfs_attach_transaction(root);
1819                 if (IS_ERR(trans)) {
1820                         if (PTR_ERR(trans) != -ENOENT)
1821                                 cannot_commit = true;
1822                         goto sleep;
1823                 }
1824                 if (transid == trans->transid) {
1825                         btrfs_commit_transaction(trans);
1826                 } else {
1827                         btrfs_end_transaction(trans);
1828                 }
1829 sleep:
1830                 wake_up_process(fs_info->cleaner_kthread);
1831                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1832
1833                 if (unlikely(test_bit(BTRFS_FS_STATE_ERROR,
1834                                       &fs_info->fs_state)))
1835                         btrfs_cleanup_transaction(fs_info);
1836                 if (!kthread_should_stop() &&
1837                                 (!btrfs_transaction_blocked(fs_info) ||
1838                                  cannot_commit))
1839                         schedule_timeout_interruptible(delay);
1840         } while (!kthread_should_stop());
1841         return 0;
1842 }
1843
1844 /*
1845  * this will find the highest generation in the array of
1846  * root backups.  The index of the highest array is returned,
1847  * or -1 if we can't find anything.
1848  *
1849  * We check to make sure the array is valid by comparing the
1850  * generation of the latest  root in the array with the generation
1851  * in the super block.  If they don't match we pitch it.
1852  */
1853 static int find_newest_super_backup(struct btrfs_fs_info *info, u64 newest_gen)
1854 {
1855         u64 cur;
1856         int newest_index = -1;
1857         struct btrfs_root_backup *root_backup;
1858         int i;
1859
1860         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1861                 root_backup = info->super_copy->super_roots + i;
1862                 cur = btrfs_backup_tree_root_gen(root_backup);
1863                 if (cur == newest_gen)
1864                         newest_index = i;
1865         }
1866
1867         /* check to see if we actually wrapped around */
1868         if (newest_index == BTRFS_NUM_BACKUP_ROOTS - 1) {
1869                 root_backup = info->super_copy->super_roots;
1870                 cur = btrfs_backup_tree_root_gen(root_backup);
1871                 if (cur == newest_gen)
1872                         newest_index = 0;
1873         }
1874         return newest_index;
1875 }
1876
1877
1878 /*
1879  * find the oldest backup so we know where to store new entries
1880  * in the backup array.  This will set the backup_root_index
1881  * field in the fs_info struct
1882  */
1883 static void find_oldest_super_backup(struct btrfs_fs_info *info,
1884                                      u64 newest_gen)
1885 {
1886         int newest_index = -1;
1887
1888         newest_index = find_newest_super_backup(info, newest_gen);
1889         /* if there was garbage in there, just move along */
1890         if (newest_index == -1) {
1891                 info->backup_root_index = 0;
1892         } else {
1893                 info->backup_root_index = (newest_index + 1) % BTRFS_NUM_BACKUP_ROOTS;
1894         }
1895 }
1896
1897 /*
1898  * copy all the root pointers into the super backup array.
1899  * this will bump the backup pointer by one when it is
1900  * done
1901  */
1902 static void backup_super_roots(struct btrfs_fs_info *info)
1903 {
1904         int next_backup;
1905         struct btrfs_root_backup *root_backup;
1906         int last_backup;
1907
1908         next_backup = info->backup_root_index;
1909         last_backup = (next_backup + BTRFS_NUM_BACKUP_ROOTS - 1) %
1910                 BTRFS_NUM_BACKUP_ROOTS;
1911
1912         /*
1913          * just overwrite the last backup if we're at the same generation
1914          * this happens only at umount
1915          */
1916         root_backup = info->super_for_commit->super_roots + last_backup;
1917         if (btrfs_backup_tree_root_gen(root_backup) ==
1918             btrfs_header_generation(info->tree_root->node))
1919                 next_backup = last_backup;
1920
1921         root_backup = info->super_for_commit->super_roots + next_backup;
1922
1923         /*
1924          * make sure all of our padding and empty slots get zero filled
1925          * regardless of which ones we use today
1926          */
1927         memset(root_backup, 0, sizeof(*root_backup));
1928
1929         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1930
1931         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1932         btrfs_set_backup_tree_root_gen(root_backup,
1933                                btrfs_header_generation(info->tree_root->node));
1934
1935         btrfs_set_backup_tree_root_level(root_backup,
1936                                btrfs_header_level(info->tree_root->node));
1937
1938         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1939         btrfs_set_backup_chunk_root_gen(root_backup,
1940                                btrfs_header_generation(info->chunk_root->node));
1941         btrfs_set_backup_chunk_root_level(root_backup,
1942                                btrfs_header_level(info->chunk_root->node));
1943
1944         btrfs_set_backup_extent_root(root_backup, info->extent_root->node->start);
1945         btrfs_set_backup_extent_root_gen(root_backup,
1946                                btrfs_header_generation(info->extent_root->node));
1947         btrfs_set_backup_extent_root_level(root_backup,
1948                                btrfs_header_level(info->extent_root->node));
1949
1950         /*
1951          * we might commit during log recovery, which happens before we set
1952          * the fs_root.  Make sure it is valid before we fill it in.
1953          */
1954         if (info->fs_root && info->fs_root->node) {
1955                 btrfs_set_backup_fs_root(root_backup,
1956                                          info->fs_root->node->start);
1957                 btrfs_set_backup_fs_root_gen(root_backup,
1958                                btrfs_header_generation(info->fs_root->node));
1959                 btrfs_set_backup_fs_root_level(root_backup,
1960                                btrfs_header_level(info->fs_root->node));
1961         }
1962
1963         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
1964         btrfs_set_backup_dev_root_gen(root_backup,
1965                                btrfs_header_generation(info->dev_root->node));
1966         btrfs_set_backup_dev_root_level(root_backup,
1967                                        btrfs_header_level(info->dev_root->node));
1968
1969         btrfs_set_backup_csum_root(root_backup, info->csum_root->node->start);
1970         btrfs_set_backup_csum_root_gen(root_backup,
1971                                btrfs_header_generation(info->csum_root->node));
1972         btrfs_set_backup_csum_root_level(root_backup,
1973                                btrfs_header_level(info->csum_root->node));
1974
1975         btrfs_set_backup_total_bytes(root_backup,
1976                              btrfs_super_total_bytes(info->super_copy));
1977         btrfs_set_backup_bytes_used(root_backup,
1978                              btrfs_super_bytes_used(info->super_copy));
1979         btrfs_set_backup_num_devices(root_backup,
1980                              btrfs_super_num_devices(info->super_copy));
1981
1982         /*
1983          * if we don't copy this out to the super_copy, it won't get remembered
1984          * for the next commit
1985          */
1986         memcpy(&info->super_copy->super_roots,
1987                &info->super_for_commit->super_roots,
1988                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
1989 }
1990
1991 /*
1992  * this copies info out of the root backup array and back into
1993  * the in-memory super block.  It is meant to help iterate through
1994  * the array, so you send it the number of backups you've already
1995  * tried and the last backup index you used.
1996  *
1997  * this returns -1 when it has tried all the backups
1998  */
1999 static noinline int next_root_backup(struct btrfs_fs_info *info,
2000                                      struct btrfs_super_block *super,
2001                                      int *num_backups_tried, int *backup_index)
2002 {
2003         struct btrfs_root_backup *root_backup;
2004         int newest = *backup_index;
2005
2006         if (*num_backups_tried == 0) {
2007                 u64 gen = btrfs_super_generation(super);
2008
2009                 newest = find_newest_super_backup(info, gen);
2010                 if (newest == -1)
2011                         return -1;
2012
2013                 *backup_index = newest;
2014                 *num_backups_tried = 1;
2015         } else if (*num_backups_tried == BTRFS_NUM_BACKUP_ROOTS) {
2016                 /* we've tried all the backups, all done */
2017                 return -1;
2018         } else {
2019                 /* jump to the next oldest backup */
2020                 newest = (*backup_index + BTRFS_NUM_BACKUP_ROOTS - 1) %
2021                         BTRFS_NUM_BACKUP_ROOTS;
2022                 *backup_index = newest;
2023                 *num_backups_tried += 1;
2024         }
2025         root_backup = super->super_roots + newest;
2026
2027         btrfs_set_super_generation(super,
2028                                    btrfs_backup_tree_root_gen(root_backup));
2029         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
2030         btrfs_set_super_root_level(super,
2031                                    btrfs_backup_tree_root_level(root_backup));
2032         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
2033
2034         /*
2035          * fixme: the total bytes and num_devices need to match or we should
2036          * need a fsck
2037          */
2038         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
2039         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
2040         return 0;
2041 }
2042
2043 /* helper to cleanup workers */
2044 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
2045 {
2046         btrfs_destroy_workqueue(fs_info->fixup_workers);
2047         btrfs_destroy_workqueue(fs_info->delalloc_workers);
2048         btrfs_destroy_workqueue(fs_info->workers);
2049         btrfs_destroy_workqueue(fs_info->endio_workers);
2050         btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
2051         btrfs_destroy_workqueue(fs_info->endio_repair_workers);
2052         btrfs_destroy_workqueue(fs_info->rmw_workers);
2053         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2054         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2055         btrfs_destroy_workqueue(fs_info->submit_workers);
2056         btrfs_destroy_workqueue(fs_info->delayed_workers);
2057         btrfs_destroy_workqueue(fs_info->caching_workers);
2058         btrfs_destroy_workqueue(fs_info->readahead_workers);
2059         btrfs_destroy_workqueue(fs_info->flush_workers);
2060         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2061         btrfs_destroy_workqueue(fs_info->extent_workers);
2062         /*
2063          * Now that all other work queues are destroyed, we can safely destroy
2064          * the queues used for metadata I/O, since tasks from those other work
2065          * queues can do metadata I/O operations.
2066          */
2067         btrfs_destroy_workqueue(fs_info->endio_meta_workers);
2068         btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
2069 }
2070
2071 static void free_root_extent_buffers(struct btrfs_root *root)
2072 {
2073         if (root) {
2074                 free_extent_buffer(root->node);
2075                 free_extent_buffer(root->commit_root);
2076                 root->node = NULL;
2077                 root->commit_root = NULL;
2078         }
2079 }
2080
2081 /* helper to cleanup tree roots */
2082 static void free_root_pointers(struct btrfs_fs_info *info, int chunk_root)
2083 {
2084         free_root_extent_buffers(info->tree_root);
2085
2086         free_root_extent_buffers(info->dev_root);
2087         free_root_extent_buffers(info->extent_root);
2088         free_root_extent_buffers(info->csum_root);
2089         free_root_extent_buffers(info->quota_root);
2090         free_root_extent_buffers(info->uuid_root);
2091         if (chunk_root)
2092                 free_root_extent_buffers(info->chunk_root);
2093         free_root_extent_buffers(info->free_space_root);
2094 }
2095
2096 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2097 {
2098         int ret;
2099         struct btrfs_root *gang[8];
2100         int i;
2101
2102         while (!list_empty(&fs_info->dead_roots)) {
2103                 gang[0] = list_entry(fs_info->dead_roots.next,
2104                                      struct btrfs_root, root_list);
2105                 list_del(&gang[0]->root_list);
2106
2107                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) {
2108                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2109                 } else {
2110                         free_extent_buffer(gang[0]->node);
2111                         free_extent_buffer(gang[0]->commit_root);
2112                         btrfs_put_fs_root(gang[0]);
2113                 }
2114         }
2115
2116         while (1) {
2117                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2118                                              (void **)gang, 0,
2119                                              ARRAY_SIZE(gang));
2120                 if (!ret)
2121                         break;
2122                 for (i = 0; i < ret; i++)
2123                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2124         }
2125
2126         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
2127                 btrfs_free_log_root_tree(NULL, fs_info);
2128                 btrfs_destroy_pinned_extent(fs_info, fs_info->pinned_extents);
2129         }
2130 }
2131
2132 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2133 {
2134         mutex_init(&fs_info->scrub_lock);
2135         atomic_set(&fs_info->scrubs_running, 0);
2136         atomic_set(&fs_info->scrub_pause_req, 0);
2137         atomic_set(&fs_info->scrubs_paused, 0);
2138         atomic_set(&fs_info->scrub_cancel_req, 0);
2139         init_waitqueue_head(&fs_info->scrub_pause_wait);
2140         fs_info->scrub_workers_refcnt = 0;
2141 }
2142
2143 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2144 {
2145         spin_lock_init(&fs_info->balance_lock);
2146         mutex_init(&fs_info->balance_mutex);
2147         atomic_set(&fs_info->balance_pause_req, 0);
2148         atomic_set(&fs_info->balance_cancel_req, 0);
2149         fs_info->balance_ctl = NULL;
2150         init_waitqueue_head(&fs_info->balance_wait_q);
2151 }
2152
2153 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2154 {
2155         struct inode *inode = fs_info->btree_inode;
2156
2157         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2158         set_nlink(inode, 1);
2159         /*
2160          * we set the i_size on the btree inode to the max possible int.
2161          * the real end of the address space is determined by all of
2162          * the devices in the system
2163          */
2164         inode->i_size = OFFSET_MAX;
2165         inode->i_mapping->a_ops = &btree_aops;
2166
2167         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2168         extent_io_tree_init(&BTRFS_I(inode)->io_tree, inode);
2169         BTRFS_I(inode)->io_tree.track_uptodate = 0;
2170         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2171
2172         BTRFS_I(inode)->io_tree.ops = &btree_extent_io_ops;
2173
2174         BTRFS_I(inode)->root = fs_info->tree_root;
2175         memset(&BTRFS_I(inode)->location, 0, sizeof(struct btrfs_key));
2176         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2177         btrfs_insert_inode_hash(inode);
2178 }
2179
2180 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2181 {
2182         fs_info->dev_replace.lock_owner = 0;
2183         atomic_set(&fs_info->dev_replace.nesting_level, 0);
2184         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2185         rwlock_init(&fs_info->dev_replace.lock);
2186         atomic_set(&fs_info->dev_replace.read_locks, 0);
2187         atomic_set(&fs_info->dev_replace.blocking_readers, 0);
2188         init_waitqueue_head(&fs_info->replace_wait);
2189         init_waitqueue_head(&fs_info->dev_replace.read_lock_wq);
2190 }
2191
2192 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2193 {
2194         spin_lock_init(&fs_info->qgroup_lock);
2195         mutex_init(&fs_info->qgroup_ioctl_lock);
2196         fs_info->qgroup_tree = RB_ROOT;
2197         fs_info->qgroup_op_tree = RB_ROOT;
2198         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2199         fs_info->qgroup_seq = 1;
2200         fs_info->qgroup_ulist = NULL;
2201         fs_info->qgroup_rescan_running = false;
2202         mutex_init(&fs_info->qgroup_rescan_lock);
2203 }
2204
2205 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info,
2206                 struct btrfs_fs_devices *fs_devices)
2207 {
2208         u32 max_active = fs_info->thread_pool_size;
2209         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2210
2211         fs_info->workers =
2212                 btrfs_alloc_workqueue(fs_info, "worker",
2213                                       flags | WQ_HIGHPRI, max_active, 16);
2214
2215         fs_info->delalloc_workers =
2216                 btrfs_alloc_workqueue(fs_info, "delalloc",
2217                                       flags, max_active, 2);
2218
2219         fs_info->flush_workers =
2220                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2221                                       flags, max_active, 0);
2222
2223         fs_info->caching_workers =
2224                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2225
2226         /*
2227          * a higher idle thresh on the submit workers makes it much more
2228          * likely that bios will be send down in a sane order to the
2229          * devices
2230          */
2231         fs_info->submit_workers =
2232                 btrfs_alloc_workqueue(fs_info, "submit", flags,
2233                                       min_t(u64, fs_devices->num_devices,
2234                                             max_active), 64);
2235
2236         fs_info->fixup_workers =
2237                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2238
2239         /*
2240          * endios are largely parallel and should have a very
2241          * low idle thresh
2242          */
2243         fs_info->endio_workers =
2244                 btrfs_alloc_workqueue(fs_info, "endio", flags, max_active, 4);
2245         fs_info->endio_meta_workers =
2246                 btrfs_alloc_workqueue(fs_info, "endio-meta", flags,
2247                                       max_active, 4);
2248         fs_info->endio_meta_write_workers =
2249                 btrfs_alloc_workqueue(fs_info, "endio-meta-write", flags,
2250                                       max_active, 2);
2251         fs_info->endio_raid56_workers =
2252                 btrfs_alloc_workqueue(fs_info, "endio-raid56", flags,
2253                                       max_active, 4);
2254         fs_info->endio_repair_workers =
2255                 btrfs_alloc_workqueue(fs_info, "endio-repair", flags, 1, 0);
2256         fs_info->rmw_workers =
2257                 btrfs_alloc_workqueue(fs_info, "rmw", flags, max_active, 2);
2258         fs_info->endio_write_workers =
2259                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2260                                       max_active, 2);
2261         fs_info->endio_freespace_worker =
2262                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2263                                       max_active, 0);
2264         fs_info->delayed_workers =
2265                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2266                                       max_active, 0);
2267         fs_info->readahead_workers =
2268                 btrfs_alloc_workqueue(fs_info, "readahead", flags,
2269                                       max_active, 2);
2270         fs_info->qgroup_rescan_workers =
2271                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2272         fs_info->extent_workers =
2273                 btrfs_alloc_workqueue(fs_info, "extent-refs", flags,
2274                                       min_t(u64, fs_devices->num_devices,
2275                                             max_active), 8);
2276
2277         if (!(fs_info->workers && fs_info->delalloc_workers &&
2278               fs_info->submit_workers && fs_info->flush_workers &&
2279               fs_info->endio_workers && fs_info->endio_meta_workers &&
2280               fs_info->endio_meta_write_workers &&
2281               fs_info->endio_repair_workers &&
2282               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2283               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2284               fs_info->caching_workers && fs_info->readahead_workers &&
2285               fs_info->fixup_workers && fs_info->delayed_workers &&
2286               fs_info->extent_workers &&
2287               fs_info->qgroup_rescan_workers)) {
2288                 return -ENOMEM;
2289         }
2290
2291         return 0;
2292 }
2293
2294 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2295                             struct btrfs_fs_devices *fs_devices)
2296 {
2297         int ret;
2298         struct btrfs_root *log_tree_root;
2299         struct btrfs_super_block *disk_super = fs_info->super_copy;
2300         u64 bytenr = btrfs_super_log_root(disk_super);
2301         int level = btrfs_super_log_root_level(disk_super);
2302
2303         if (fs_devices->rw_devices == 0) {
2304                 btrfs_warn(fs_info, "log replay required on RO media");
2305                 return -EIO;
2306         }
2307
2308         log_tree_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2309         if (!log_tree_root)
2310                 return -ENOMEM;
2311
2312         __setup_root(log_tree_root, fs_info, BTRFS_TREE_LOG_OBJECTID);
2313
2314         log_tree_root->node = read_tree_block(fs_info, bytenr,
2315                                               fs_info->generation + 1,
2316                                               level, NULL);
2317         if (IS_ERR(log_tree_root->node)) {
2318                 btrfs_warn(fs_info, "failed to read log tree");
2319                 ret = PTR_ERR(log_tree_root->node);
2320                 kfree(log_tree_root);
2321                 return ret;
2322         } else if (!extent_buffer_uptodate(log_tree_root->node)) {
2323                 btrfs_err(fs_info, "failed to read log tree");
2324                 free_extent_buffer(log_tree_root->node);
2325                 kfree(log_tree_root);
2326                 return -EIO;
2327         }
2328         /* returns with log_tree_root freed on success */
2329         ret = btrfs_recover_log_trees(log_tree_root);
2330         if (ret) {
2331                 btrfs_handle_fs_error(fs_info, ret,
2332                                       "Failed to recover log tree");
2333                 free_extent_buffer(log_tree_root->node);
2334                 kfree(log_tree_root);
2335                 return ret;
2336         }
2337
2338         if (sb_rdonly(fs_info->sb)) {
2339                 ret = btrfs_commit_super(fs_info);
2340                 if (ret)
2341                         return ret;
2342         }
2343
2344         return 0;
2345 }
2346
2347 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2348 {
2349         struct btrfs_root *tree_root = fs_info->tree_root;
2350         struct btrfs_root *root;
2351         struct btrfs_key location;
2352         int ret;
2353
2354         BUG_ON(!fs_info->tree_root);
2355
2356         location.objectid = BTRFS_EXTENT_TREE_OBJECTID;
2357         location.type = BTRFS_ROOT_ITEM_KEY;
2358         location.offset = 0;
2359
2360         root = btrfs_read_tree_root(tree_root, &location);
2361         if (IS_ERR(root)) {
2362                 ret = PTR_ERR(root);
2363                 goto out;
2364         }
2365         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2366         fs_info->extent_root = root;
2367
2368         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2369         root = btrfs_read_tree_root(tree_root, &location);
2370         if (IS_ERR(root)) {
2371                 ret = PTR_ERR(root);
2372                 goto out;
2373         }
2374         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2375         fs_info->dev_root = root;
2376         btrfs_init_devices_late(fs_info);
2377
2378         location.objectid = BTRFS_CSUM_TREE_OBJECTID;
2379         root = btrfs_read_tree_root(tree_root, &location);
2380         if (IS_ERR(root)) {
2381                 ret = PTR_ERR(root);
2382                 goto out;
2383         }
2384         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2385         fs_info->csum_root = root;
2386
2387         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2388         root = btrfs_read_tree_root(tree_root, &location);
2389         if (!IS_ERR(root)) {
2390                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2391                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2392                 fs_info->quota_root = root;
2393         }
2394
2395         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2396         root = btrfs_read_tree_root(tree_root, &location);
2397         if (IS_ERR(root)) {
2398                 ret = PTR_ERR(root);
2399                 if (ret != -ENOENT)
2400                         goto out;
2401         } else {
2402                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2403                 fs_info->uuid_root = root;
2404         }
2405
2406         if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
2407                 location.objectid = BTRFS_FREE_SPACE_TREE_OBJECTID;
2408                 root = btrfs_read_tree_root(tree_root, &location);
2409                 if (IS_ERR(root)) {
2410                         ret = PTR_ERR(root);
2411                         goto out;
2412                 }
2413                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2414                 fs_info->free_space_root = root;
2415         }
2416
2417         return 0;
2418 out:
2419         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2420                    location.objectid, ret);
2421         return ret;
2422 }
2423
2424 /*
2425  * Real super block validation
2426  * NOTE: super csum type and incompat features will not be checked here.
2427  *
2428  * @sb:         super block to check
2429  * @mirror_num: the super block number to check its bytenr:
2430  *              0       the primary (1st) sb
2431  *              1, 2    2nd and 3rd backup copy
2432  *             -1       skip bytenr check
2433  */
2434 static int validate_super(struct btrfs_fs_info *fs_info,
2435                             struct btrfs_super_block *sb, int mirror_num)
2436 {
2437         u64 nodesize = btrfs_super_nodesize(sb);
2438         u64 sectorsize = btrfs_super_sectorsize(sb);
2439         int ret = 0;
2440
2441         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2442                 btrfs_err(fs_info, "no valid FS found");
2443                 ret = -EINVAL;
2444         }
2445         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2446                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2447                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2448                 ret = -EINVAL;
2449         }
2450         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2451                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2452                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2453                 ret = -EINVAL;
2454         }
2455         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2456                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2457                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2458                 ret = -EINVAL;
2459         }
2460         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2461                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2462                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2463                 ret = -EINVAL;
2464         }
2465
2466         /*
2467          * Check sectorsize and nodesize first, other check will need it.
2468          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2469          */
2470         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2471             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2472                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2473                 ret = -EINVAL;
2474         }
2475         /* Only PAGE SIZE is supported yet */
2476         if (sectorsize != PAGE_SIZE) {
2477                 btrfs_err(fs_info,
2478                         "sectorsize %llu not supported yet, only support %lu",
2479                         sectorsize, PAGE_SIZE);
2480                 ret = -EINVAL;
2481         }
2482         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2483             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2484                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2485                 ret = -EINVAL;
2486         }
2487         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2488                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2489                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2490                 ret = -EINVAL;
2491         }
2492
2493         /* Root alignment check */
2494         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2495                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2496                            btrfs_super_root(sb));
2497                 ret = -EINVAL;
2498         }
2499         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2500                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2501                            btrfs_super_chunk_root(sb));
2502                 ret = -EINVAL;
2503         }
2504         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2505                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2506                            btrfs_super_log_root(sb));
2507                 ret = -EINVAL;
2508         }
2509
2510         if (memcmp(fs_info->fsid, sb->dev_item.fsid, BTRFS_FSID_SIZE) != 0) {
2511                 btrfs_err(fs_info,
2512                            "dev_item UUID does not match fsid: %pU != %pU",
2513                            fs_info->fsid, sb->dev_item.fsid);
2514                 ret = -EINVAL;
2515         }
2516
2517         /*
2518          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2519          * done later
2520          */
2521         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2522                 btrfs_err(fs_info, "bytes_used is too small %llu",
2523                           btrfs_super_bytes_used(sb));
2524                 ret = -EINVAL;
2525         }
2526         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2527                 btrfs_err(fs_info, "invalid stripesize %u",
2528                           btrfs_super_stripesize(sb));
2529                 ret = -EINVAL;
2530         }
2531         if (btrfs_super_num_devices(sb) > (1UL << 31))
2532                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2533                            btrfs_super_num_devices(sb));
2534         if (btrfs_super_num_devices(sb) == 0) {
2535                 btrfs_err(fs_info, "number of devices is 0");
2536                 ret = -EINVAL;
2537         }
2538
2539         if (mirror_num >= 0 &&
2540             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2541                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2542                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2543                 ret = -EINVAL;
2544         }
2545
2546         /*
2547          * Obvious sys_chunk_array corruptions, it must hold at least one key
2548          * and one chunk
2549          */
2550         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2551                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2552                           btrfs_super_sys_array_size(sb),
2553                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2554                 ret = -EINVAL;
2555         }
2556         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2557                         + sizeof(struct btrfs_chunk)) {
2558                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2559                           btrfs_super_sys_array_size(sb),
2560                           sizeof(struct btrfs_disk_key)
2561                           + sizeof(struct btrfs_chunk));
2562                 ret = -EINVAL;
2563         }
2564
2565         /*
2566          * The generation is a global counter, we'll trust it more than the others
2567          * but it's still possible that it's the one that's wrong.
2568          */
2569         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2570                 btrfs_warn(fs_info,
2571                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2572                         btrfs_super_generation(sb),
2573                         btrfs_super_chunk_root_generation(sb));
2574         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2575             && btrfs_super_cache_generation(sb) != (u64)-1)
2576                 btrfs_warn(fs_info,
2577                         "suspicious: generation < cache_generation: %llu < %llu",
2578                         btrfs_super_generation(sb),
2579                         btrfs_super_cache_generation(sb));
2580
2581         return ret;
2582 }
2583
2584 /*
2585  * Validation of super block at mount time.
2586  * Some checks already done early at mount time, like csum type and incompat
2587  * flags will be skipped.
2588  */
2589 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2590 {
2591         return validate_super(fs_info, fs_info->super_copy, 0);
2592 }
2593
2594 /*
2595  * Validation of super block at write time.
2596  * Some checks like bytenr check will be skipped as their values will be
2597  * overwritten soon.
2598  * Extra checks like csum type and incompat flags will be done here.
2599  */
2600 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2601                                       struct btrfs_super_block *sb)
2602 {
2603         int ret;
2604
2605         ret = validate_super(fs_info, sb, -1);
2606         if (ret < 0)
2607                 goto out;
2608         if (btrfs_super_csum_type(sb) != BTRFS_CSUM_TYPE_CRC32) {
2609                 ret = -EUCLEAN;
2610                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2611                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2612                 goto out;
2613         }
2614         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2615                 ret = -EUCLEAN;
2616                 btrfs_err(fs_info,
2617                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2618                           btrfs_super_incompat_flags(sb),
2619                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2620                 goto out;
2621         }
2622 out:
2623         if (ret < 0)
2624                 btrfs_err(fs_info,
2625                 "super block corruption detected before writing it to disk");
2626         return ret;
2627 }
2628
2629 int open_ctree(struct super_block *sb,
2630                struct btrfs_fs_devices *fs_devices,
2631                char *options)
2632 {
2633         u32 sectorsize;
2634         u32 nodesize;
2635         u32 stripesize;
2636         u64 generation;
2637         u64 features;
2638         struct btrfs_key location;
2639         struct buffer_head *bh;
2640         struct btrfs_super_block *disk_super;
2641         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
2642         struct btrfs_root *tree_root;
2643         struct btrfs_root *chunk_root;
2644         int ret;
2645         int err = -EINVAL;
2646         int num_backups_tried = 0;
2647         int backup_index = 0;
2648         int clear_free_space_tree = 0;
2649         int level;
2650
2651         tree_root = fs_info->tree_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2652         chunk_root = fs_info->chunk_root = btrfs_alloc_root(fs_info, GFP_KERNEL);
2653         if (!tree_root || !chunk_root) {
2654                 err = -ENOMEM;
2655                 goto fail;
2656         }
2657
2658         ret = init_srcu_struct(&fs_info->subvol_srcu);
2659         if (ret) {
2660                 err = ret;
2661                 goto fail;
2662         }
2663
2664         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
2665         if (ret) {
2666                 err = ret;
2667                 goto fail_srcu;
2668         }
2669         fs_info->dirty_metadata_batch = PAGE_SIZE *
2670                                         (1 + ilog2(nr_cpu_ids));
2671
2672         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
2673         if (ret) {
2674                 err = ret;
2675                 goto fail_dirty_metadata_bytes;
2676         }
2677
2678         ret = percpu_counter_init(&fs_info->bio_counter, 0, GFP_KERNEL);
2679         if (ret) {
2680                 err = ret;
2681                 goto fail_delalloc_bytes;
2682         }
2683
2684         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2685         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2686         INIT_LIST_HEAD(&fs_info->trans_list);
2687         INIT_LIST_HEAD(&fs_info->dead_roots);
2688         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2689         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2690         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2691         INIT_LIST_HEAD(&fs_info->pending_raid_kobjs);
2692         spin_lock_init(&fs_info->pending_raid_kobjs_lock);
2693         spin_lock_init(&fs_info->delalloc_root_lock);
2694         spin_lock_init(&fs_info->trans_lock);
2695         spin_lock_init(&fs_info->fs_roots_radix_lock);
2696         spin_lock_init(&fs_info->delayed_iput_lock);
2697         spin_lock_init(&fs_info->defrag_inodes_lock);
2698         spin_lock_init(&fs_info->tree_mod_seq_lock);
2699         spin_lock_init(&fs_info->super_lock);
2700         spin_lock_init(&fs_info->qgroup_op_lock);
2701         spin_lock_init(&fs_info->buffer_lock);
2702         spin_lock_init(&fs_info->unused_bgs_lock);
2703         rwlock_init(&fs_info->tree_mod_log_lock);
2704         mutex_init(&fs_info->unused_bg_unpin_mutex);
2705         mutex_init(&fs_info->delete_unused_bgs_mutex);
2706         mutex_init(&fs_info->reloc_mutex);
2707         mutex_init(&fs_info->delalloc_root_mutex);
2708         mutex_init(&fs_info->cleaner_delayed_iput_mutex);
2709         seqlock_init(&fs_info->profiles_lock);
2710
2711         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
2712         INIT_LIST_HEAD(&fs_info->space_info);
2713         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
2714         INIT_LIST_HEAD(&fs_info->unused_bgs);
2715         btrfs_mapping_init(&fs_info->mapping_tree);
2716         btrfs_init_block_rsv(&fs_info->global_block_rsv,
2717                              BTRFS_BLOCK_RSV_GLOBAL);
2718         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
2719         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
2720         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
2721         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
2722                              BTRFS_BLOCK_RSV_DELOPS);
2723         atomic_set(&fs_info->async_delalloc_pages, 0);
2724         atomic_set(&fs_info->defrag_running, 0);
2725         atomic_set(&fs_info->qgroup_op_seq, 0);
2726         atomic_set(&fs_info->reada_works_cnt, 0);
2727         atomic64_set(&fs_info->tree_mod_seq, 0);
2728         fs_info->sb = sb;
2729         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
2730         fs_info->metadata_ratio = 0;
2731         fs_info->defrag_inodes = RB_ROOT;
2732         atomic64_set(&fs_info->free_chunk_space, 0);
2733         fs_info->tree_mod_log = RB_ROOT;
2734         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
2735         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
2736         /* readahead state */
2737         INIT_RADIX_TREE(&fs_info->reada_tree, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
2738         spin_lock_init(&fs_info->reada_lock);
2739         btrfs_init_ref_verify(fs_info);
2740
2741         fs_info->thread_pool_size = min_t(unsigned long,
2742                                           num_online_cpus() + 2, 8);
2743
2744         INIT_LIST_HEAD(&fs_info->ordered_roots);
2745         spin_lock_init(&fs_info->ordered_root_lock);
2746
2747         fs_info->btree_inode = new_inode(sb);
2748         if (!fs_info->btree_inode) {
2749                 err = -ENOMEM;
2750                 goto fail_bio_counter;
2751         }
2752         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
2753
2754         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
2755                                         GFP_KERNEL);
2756         if (!fs_info->delayed_root) {
2757                 err = -ENOMEM;
2758                 goto fail_iput;
2759         }
2760         btrfs_init_delayed_root(fs_info->delayed_root);
2761
2762         btrfs_init_scrub(fs_info);
2763 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
2764         fs_info->check_integrity_print_mask = 0;
2765 #endif
2766         btrfs_init_balance(fs_info);
2767         btrfs_init_async_reclaim_work(&fs_info->async_reclaim_work);
2768
2769         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
2770         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
2771
2772         btrfs_init_btree_inode(fs_info);
2773
2774         spin_lock_init(&fs_info->block_group_cache_lock);
2775         fs_info->block_group_cache_tree = RB_ROOT;
2776         fs_info->first_logical_byte = (u64)-1;
2777
2778         extent_io_tree_init(&fs_info->freed_extents[0], NULL);
2779         extent_io_tree_init(&fs_info->freed_extents[1], NULL);
2780         fs_info->pinned_extents = &fs_info->freed_extents[0];
2781         set_bit(BTRFS_FS_BARRIER, &fs_info->flags);
2782
2783         mutex_init(&fs_info->ordered_operations_mutex);
2784         mutex_init(&fs_info->tree_log_mutex);
2785         mutex_init(&fs_info->chunk_mutex);
2786         mutex_init(&fs_info->transaction_kthread_mutex);
2787         mutex_init(&fs_info->cleaner_mutex);
2788         mutex_init(&fs_info->ro_block_group_mutex);
2789         init_rwsem(&fs_info->commit_root_sem);
2790         init_rwsem(&fs_info->cleanup_work_sem);
2791         init_rwsem(&fs_info->subvol_sem);
2792         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
2793
2794         btrfs_init_dev_replace_locks(fs_info);
2795         btrfs_init_qgroup(fs_info);
2796
2797         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
2798         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
2799
2800         init_waitqueue_head(&fs_info->transaction_throttle);
2801         init_waitqueue_head(&fs_info->transaction_wait);
2802         init_waitqueue_head(&fs_info->transaction_blocked_wait);
2803         init_waitqueue_head(&fs_info->async_submit_wait);
2804
2805         INIT_LIST_HEAD(&fs_info->pinned_chunks);
2806
2807         /* Usable values until the real ones are cached from the superblock */
2808         fs_info->nodesize = 4096;
2809         fs_info->sectorsize = 4096;
2810         fs_info->stripesize = 4096;
2811
2812         ret = btrfs_alloc_stripe_hash_table(fs_info);
2813         if (ret) {
2814                 err = ret;
2815                 goto fail_alloc;
2816         }
2817
2818         __setup_root(tree_root, fs_info, BTRFS_ROOT_TREE_OBJECTID);
2819
2820         invalidate_bdev(fs_devices->latest_bdev);
2821
2822         /*
2823          * Read super block and check the signature bytes only
2824          */
2825         bh = btrfs_read_dev_super(fs_devices->latest_bdev);
2826         if (IS_ERR(bh)) {
2827                 err = PTR_ERR(bh);
2828                 goto fail_alloc;
2829         }
2830
2831         /*
2832          * We want to check superblock checksum, the type is stored inside.
2833          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
2834          */
2835         if (btrfs_check_super_csum(fs_info, bh->b_data)) {
2836                 btrfs_err(fs_info, "superblock checksum mismatch");
2837                 err = -EINVAL;
2838                 brelse(bh);
2839                 goto fail_alloc;
2840         }
2841
2842         /*
2843          * super_copy is zeroed at allocation time and we never touch the
2844          * following bytes up to INFO_SIZE, the checksum is calculated from
2845          * the whole block of INFO_SIZE
2846          */
2847         memcpy(fs_info->super_copy, bh->b_data, sizeof(*fs_info->super_copy));
2848         memcpy(fs_info->super_for_commit, fs_info->super_copy,
2849                sizeof(*fs_info->super_for_commit));
2850         brelse(bh);
2851
2852         memcpy(fs_info->fsid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE);
2853
2854         ret = btrfs_validate_mount_super(fs_info);
2855         if (ret) {
2856                 btrfs_err(fs_info, "superblock contains fatal errors");
2857                 err = -EINVAL;
2858                 goto fail_alloc;
2859         }
2860
2861         disk_super = fs_info->super_copy;
2862         if (!btrfs_super_root(disk_super))
2863                 goto fail_alloc;
2864
2865         /* check FS state, whether FS is broken. */
2866         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
2867                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
2868
2869         /*
2870          * run through our array of backup supers and setup
2871          * our ring pointer to the oldest one
2872          */
2873         generation = btrfs_super_generation(disk_super);
2874         find_oldest_super_backup(fs_info, generation);
2875
2876         /*
2877          * In the long term, we'll store the compression type in the super
2878          * block, and it'll be used for per file compression control.
2879          */
2880         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
2881
2882         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
2883         if (ret) {
2884                 err = ret;
2885                 goto fail_alloc;
2886         }
2887
2888         features = btrfs_super_incompat_flags(disk_super) &
2889                 ~BTRFS_FEATURE_INCOMPAT_SUPP;
2890         if (features) {
2891                 btrfs_err(fs_info,
2892                     "cannot mount because of unsupported optional features (%llx)",
2893                     features);
2894                 err = -EINVAL;
2895                 goto fail_alloc;
2896         }
2897
2898         features = btrfs_super_incompat_flags(disk_super);
2899         features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
2900         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
2901                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
2902         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
2903                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
2904
2905         if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
2906                 btrfs_info(fs_info, "has skinny extents");
2907
2908         /*
2909          * flag our filesystem as having big metadata blocks if
2910          * they are bigger than the page size
2911          */
2912         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) {
2913                 if (!(features & BTRFS_FEATURE_INCOMPAT_BIG_METADATA))
2914                         btrfs_info(fs_info,
2915                                 "flagging fs with big metadata feature");
2916                 features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
2917         }
2918
2919         nodesize = btrfs_super_nodesize(disk_super);
2920         sectorsize = btrfs_super_sectorsize(disk_super);
2921         stripesize = sectorsize;
2922         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
2923         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
2924
2925         /* Cache block sizes */
2926         fs_info->nodesize = nodesize;
2927         fs_info->sectorsize = sectorsize;
2928         fs_info->stripesize = stripesize;
2929
2930         /*
2931          * mixed block groups end up with duplicate but slightly offset
2932          * extent buffers for the same range.  It leads to corruptions
2933          */
2934         if ((features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
2935             (sectorsize != nodesize)) {
2936                 btrfs_err(fs_info,
2937 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
2938                         nodesize, sectorsize);
2939                 goto fail_alloc;
2940         }
2941
2942         /*
2943          * Needn't use the lock because there is no other task which will
2944          * update the flag.
2945          */
2946         btrfs_set_super_incompat_flags(disk_super, features);
2947
2948         features = btrfs_super_compat_ro_flags(disk_super) &
2949                 ~BTRFS_FEATURE_COMPAT_RO_SUPP;
2950         if (!sb_rdonly(sb) && features) {
2951                 btrfs_err(fs_info,
2952         "cannot mount read-write because of unsupported optional features (%llx)",
2953                        features);
2954                 err = -EINVAL;
2955                 goto fail_alloc;
2956         }
2957
2958         ret = btrfs_init_workqueues(fs_info, fs_devices);
2959         if (ret) {
2960                 err = ret;
2961                 goto fail_sb_buffer;
2962         }
2963
2964         sb->s_bdi->congested_fn = btrfs_congested_fn;
2965         sb->s_bdi->congested_data = fs_info;
2966         sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK;
2967         sb->s_bdi->ra_pages = VM_MAX_READAHEAD * SZ_1K / PAGE_SIZE;
2968         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
2969         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
2970
2971         sb->s_blocksize = sectorsize;
2972         sb->s_blocksize_bits = blksize_bits(sectorsize);
2973         memcpy(&sb->s_uuid, fs_info->fsid, BTRFS_FSID_SIZE);
2974
2975         mutex_lock(&fs_info->chunk_mutex);
2976         ret = btrfs_read_sys_array(fs_info);
2977         mutex_unlock(&fs_info->chunk_mutex);
2978         if (ret) {
2979                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
2980                 goto fail_sb_buffer;
2981         }
2982
2983         generation = btrfs_super_chunk_root_generation(disk_super);
2984         level = btrfs_super_chunk_root_level(disk_super);
2985
2986         __setup_root(chunk_root, fs_info, BTRFS_CHUNK_TREE_OBJECTID);
2987
2988         chunk_root->node = read_tree_block(fs_info,
2989                                            btrfs_super_chunk_root(disk_super),
2990                                            generation, level, NULL);
2991         if (IS_ERR(chunk_root->node) ||
2992             !extent_buffer_uptodate(chunk_root->node)) {
2993                 btrfs_err(fs_info, "failed to read chunk root");
2994                 if (!IS_ERR(chunk_root->node))
2995                         free_extent_buffer(chunk_root->node);
2996                 chunk_root->node = NULL;
2997                 goto fail_tree_roots;
2998         }
2999         btrfs_set_root_node(&chunk_root->root_item, chunk_root->node);
3000         chunk_root->commit_root = btrfs_root_node(chunk_root);
3001
3002         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3003            btrfs_header_chunk_tree_uuid(chunk_root->node), BTRFS_UUID_SIZE);
3004
3005         ret = btrfs_read_chunk_tree(fs_info);
3006         if (ret) {
3007                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3008                 goto fail_tree_roots;
3009         }
3010
3011         /*
3012          * Keep the devid that is marked to be the target device for the
3013          * device replace procedure
3014          */
3015         btrfs_free_extra_devids(fs_devices, 0);
3016
3017         if (!fs_devices->latest_bdev) {
3018                 btrfs_err(fs_info, "failed to read devices");
3019                 goto fail_tree_roots;
3020         }
3021
3022 retry_root_backup:
3023         generation = btrfs_super_generation(disk_super);
3024         level = btrfs_super_root_level(disk_super);
3025
3026         tree_root->node = read_tree_block(fs_info,
3027                                           btrfs_super_root(disk_super),
3028                                           generation, level, NULL);
3029         if (IS_ERR(tree_root->node) ||
3030             !extent_buffer_uptodate(tree_root->node)) {
3031                 btrfs_warn(fs_info, "failed to read tree root");
3032                 if (!IS_ERR(tree_root->node))
3033                         free_extent_buffer(tree_root->node);
3034                 tree_root->node = NULL;
3035                 goto recovery_tree_root;
3036         }
3037
3038         btrfs_set_root_node(&tree_root->root_item, tree_root->node);
3039         tree_root->commit_root = btrfs_root_node(tree_root);
3040         btrfs_set_root_refs(&tree_root->root_item, 1);
3041
3042         mutex_lock(&tree_root->objectid_mutex);
3043         ret = btrfs_find_highest_objectid(tree_root,
3044                                         &tree_root->highest_objectid);
3045         if (ret) {
3046                 mutex_unlock(&tree_root->objectid_mutex);
3047                 goto recovery_tree_root;
3048         }
3049
3050         ASSERT(tree_root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
3051
3052         mutex_unlock(&tree_root->objectid_mutex);
3053
3054         ret = btrfs_read_roots(fs_info);
3055         if (ret)
3056                 goto recovery_tree_root;
3057
3058         fs_info->generation = generation;
3059         fs_info->last_trans_committed = generation;
3060
3061         ret = btrfs_recover_balance(fs_info);
3062         if (ret) {
3063                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3064                 goto fail_block_groups;
3065         }
3066
3067         ret = btrfs_init_dev_stats(fs_info);
3068         if (ret) {
3069                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3070                 goto fail_block_groups;
3071         }
3072
3073         ret = btrfs_init_dev_replace(fs_info);
3074         if (ret) {
3075                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3076                 goto fail_block_groups;
3077         }
3078
3079         btrfs_free_extra_devids(fs_devices, 1);
3080
3081         ret = btrfs_sysfs_add_fsid(fs_devices, NULL);
3082         if (ret) {
3083                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3084                                 ret);
3085                 goto fail_block_groups;
3086         }
3087
3088         ret = btrfs_sysfs_add_device(fs_devices);
3089         if (ret) {
3090                 btrfs_err(fs_info, "failed to init sysfs device interface: %d",
3091                                 ret);
3092                 goto fail_fsdev_sysfs;
3093         }
3094
3095         ret = btrfs_sysfs_add_mounted(fs_info);
3096         if (ret) {
3097                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3098                 goto fail_fsdev_sysfs;
3099         }
3100
3101         ret = btrfs_init_space_info(fs_info);
3102         if (ret) {
3103                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3104                 goto fail_sysfs;
3105         }
3106
3107         ret = btrfs_read_block_groups(fs_info);
3108         if (ret) {
3109                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3110                 goto fail_sysfs;
3111         }
3112
3113         if (!sb_rdonly(sb) && !btrfs_check_rw_degradable(fs_info, NULL)) {
3114                 btrfs_warn(fs_info,
3115                 "writeable mount is not allowed due to too many missing devices");
3116                 goto fail_sysfs;
3117         }
3118
3119         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, tree_root,
3120                                                "btrfs-cleaner");
3121         if (IS_ERR(fs_info->cleaner_kthread))
3122                 goto fail_sysfs;
3123
3124         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3125                                                    tree_root,
3126                                                    "btrfs-transaction");
3127         if (IS_ERR(fs_info->transaction_kthread))
3128                 goto fail_cleaner;
3129
3130         if (!btrfs_test_opt(fs_info, NOSSD) &&
3131             !fs_info->fs_devices->rotating) {
3132                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3133         }
3134
3135         /*
3136          * Mount does not set all options immediately, we can do it now and do
3137          * not have to wait for transaction commit
3138          */
3139         btrfs_apply_pending_changes(fs_info);
3140
3141 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3142         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3143                 ret = btrfsic_mount(fs_info, fs_devices,
3144                                     btrfs_test_opt(fs_info,
3145                                         CHECK_INTEGRITY_INCLUDING_EXTENT_DATA) ?
3146                                     1 : 0,
3147                                     fs_info->check_integrity_print_mask);
3148                 if (ret)
3149                         btrfs_warn(fs_info,
3150                                 "failed to initialize integrity check module: %d",
3151                                 ret);
3152         }
3153 #endif
3154         ret = btrfs_read_qgroup_config(fs_info);
3155         if (ret)
3156                 goto fail_trans_kthread;
3157
3158         if (btrfs_build_ref_tree(fs_info))
3159                 btrfs_err(fs_info, "couldn't build ref tree");
3160
3161         /* do not make disk changes in broken FS or nologreplay is given */
3162         if (btrfs_super_log_root(disk_super) != 0 &&
3163             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3164                 ret = btrfs_replay_log(fs_info, fs_devices);
3165                 if (ret) {
3166                         err = ret;
3167                         goto fail_qgroup;
3168                 }
3169         }
3170
3171         ret = btrfs_find_orphan_roots(fs_info);
3172         if (ret)
3173                 goto fail_qgroup;
3174
3175         if (!sb_rdonly(sb)) {
3176                 ret = btrfs_cleanup_fs_roots(fs_info);
3177                 if (ret)
3178                         goto fail_qgroup;
3179
3180                 mutex_lock(&fs_info->cleaner_mutex);
3181                 ret = btrfs_recover_relocation(tree_root);
3182                 mutex_unlock(&fs_info->cleaner_mutex);
3183                 if (ret < 0) {
3184                         btrfs_warn(fs_info, "failed to recover relocation: %d",
3185                                         ret);
3186                         err = -EINVAL;
3187                         goto fail_qgroup;
3188                 }
3189         }
3190
3191         location.objectid = BTRFS_FS_TREE_OBJECTID;
3192         location.type = BTRFS_ROOT_ITEM_KEY;
3193         location.offset = 0;
3194
3195         fs_info->fs_root = btrfs_read_fs_root_no_name(fs_info, &location);
3196         if (IS_ERR(fs_info->fs_root)) {
3197                 err = PTR_ERR(fs_info->fs_root);
3198                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3199                 goto fail_qgroup;
3200         }
3201
3202         if (sb_rdonly(sb))
3203                 return 0;
3204
3205         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3206             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3207                 clear_free_space_tree = 1;
3208         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3209                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3210                 btrfs_warn(fs_info, "free space tree is invalid");
3211                 clear_free_space_tree = 1;
3212         }
3213
3214         if (clear_free_space_tree) {
3215                 btrfs_info(fs_info, "clearing free space tree");
3216                 ret = btrfs_clear_free_space_tree(fs_info);
3217                 if (ret) {
3218                         btrfs_warn(fs_info,
3219                                    "failed to clear free space tree: %d", ret);
3220                         close_ctree(fs_info);
3221                         return ret;
3222                 }
3223         }
3224
3225         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3226             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3227                 btrfs_info(fs_info, "creating free space tree");
3228                 ret = btrfs_create_free_space_tree(fs_info);
3229                 if (ret) {
3230                         btrfs_warn(fs_info,
3231                                 "failed to create free space tree: %d", ret);
3232                         close_ctree(fs_info);
3233                         return ret;
3234                 }
3235         }
3236
3237         down_read(&fs_info->cleanup_work_sem);
3238         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3239             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3240                 up_read(&fs_info->cleanup_work_sem);
3241                 close_ctree(fs_info);
3242                 return ret;
3243         }
3244         up_read(&fs_info->cleanup_work_sem);
3245
3246         ret = btrfs_resume_balance_async(fs_info);
3247         if (ret) {
3248                 btrfs_warn(fs_info, "failed to resume balance: %d", ret);
3249                 close_ctree(fs_info);
3250                 return ret;
3251         }
3252
3253         ret = btrfs_resume_dev_replace_async(fs_info);
3254         if (ret) {
3255                 btrfs_warn(fs_info, "failed to resume device replace: %d", ret);
3256                 close_ctree(fs_info);
3257                 return ret;