btrfs: tree-checker: Verify dev item
[sfrench/cifs-2.6.git] / fs / btrfs / volumes.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/ratelimit.h>
12 #include <linux/kthread.h>
13 #include <linux/raid/pq.h>
14 #include <linux/semaphore.h>
15 #include <linux/uuid.h>
16 #include <linux/list_sort.h>
17 #include "ctree.h"
18 #include "extent_map.h"
19 #include "disk-io.h"
20 #include "transaction.h"
21 #include "print-tree.h"
22 #include "volumes.h"
23 #include "raid56.h"
24 #include "async-thread.h"
25 #include "check-integrity.h"
26 #include "rcu-string.h"
27 #include "math.h"
28 #include "dev-replace.h"
29 #include "sysfs.h"
30 #include "tree-checker.h"
31
32 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
33         [BTRFS_RAID_RAID10] = {
34                 .sub_stripes    = 2,
35                 .dev_stripes    = 1,
36                 .devs_max       = 0,    /* 0 == as many as possible */
37                 .devs_min       = 4,
38                 .tolerated_failures = 1,
39                 .devs_increment = 2,
40                 .ncopies        = 2,
41                 .nparity        = 0,
42                 .raid_name      = "raid10",
43                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID10,
44                 .mindev_error   = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
45         },
46         [BTRFS_RAID_RAID1] = {
47                 .sub_stripes    = 1,
48                 .dev_stripes    = 1,
49                 .devs_max       = 2,
50                 .devs_min       = 2,
51                 .tolerated_failures = 1,
52                 .devs_increment = 2,
53                 .ncopies        = 2,
54                 .nparity        = 0,
55                 .raid_name      = "raid1",
56                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1,
57                 .mindev_error   = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
58         },
59         [BTRFS_RAID_DUP] = {
60                 .sub_stripes    = 1,
61                 .dev_stripes    = 2,
62                 .devs_max       = 1,
63                 .devs_min       = 1,
64                 .tolerated_failures = 0,
65                 .devs_increment = 1,
66                 .ncopies        = 2,
67                 .nparity        = 0,
68                 .raid_name      = "dup",
69                 .bg_flag        = BTRFS_BLOCK_GROUP_DUP,
70                 .mindev_error   = 0,
71         },
72         [BTRFS_RAID_RAID0] = {
73                 .sub_stripes    = 1,
74                 .dev_stripes    = 1,
75                 .devs_max       = 0,
76                 .devs_min       = 2,
77                 .tolerated_failures = 0,
78                 .devs_increment = 1,
79                 .ncopies        = 1,
80                 .nparity        = 0,
81                 .raid_name      = "raid0",
82                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID0,
83                 .mindev_error   = 0,
84         },
85         [BTRFS_RAID_SINGLE] = {
86                 .sub_stripes    = 1,
87                 .dev_stripes    = 1,
88                 .devs_max       = 1,
89                 .devs_min       = 1,
90                 .tolerated_failures = 0,
91                 .devs_increment = 1,
92                 .ncopies        = 1,
93                 .nparity        = 0,
94                 .raid_name      = "single",
95                 .bg_flag        = 0,
96                 .mindev_error   = 0,
97         },
98         [BTRFS_RAID_RAID5] = {
99                 .sub_stripes    = 1,
100                 .dev_stripes    = 1,
101                 .devs_max       = 0,
102                 .devs_min       = 2,
103                 .tolerated_failures = 1,
104                 .devs_increment = 1,
105                 .ncopies        = 1,
106                 .nparity        = 1,
107                 .raid_name      = "raid5",
108                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID5,
109                 .mindev_error   = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
110         },
111         [BTRFS_RAID_RAID6] = {
112                 .sub_stripes    = 1,
113                 .dev_stripes    = 1,
114                 .devs_max       = 0,
115                 .devs_min       = 3,
116                 .tolerated_failures = 2,
117                 .devs_increment = 1,
118                 .ncopies        = 1,
119                 .nparity        = 2,
120                 .raid_name      = "raid6",
121                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID6,
122                 .mindev_error   = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
123         },
124 };
125
126 const char *get_raid_name(enum btrfs_raid_types type)
127 {
128         if (type >= BTRFS_NR_RAID_TYPES)
129                 return NULL;
130
131         return btrfs_raid_array[type].raid_name;
132 }
133
134 /*
135  * Fill @buf with textual description of @bg_flags, no more than @size_buf
136  * bytes including terminating null byte.
137  */
138 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
139 {
140         int i;
141         int ret;
142         char *bp = buf;
143         u64 flags = bg_flags;
144         u32 size_bp = size_buf;
145
146         if (!flags) {
147                 strcpy(bp, "NONE");
148                 return;
149         }
150
151 #define DESCRIBE_FLAG(flag, desc)                                               \
152         do {                                                            \
153                 if (flags & (flag)) {                                   \
154                         ret = snprintf(bp, size_bp, "%s|", (desc));     \
155                         if (ret < 0 || ret >= size_bp)                  \
156                                 goto out_overflow;                      \
157                         size_bp -= ret;                                 \
158                         bp += ret;                                      \
159                         flags &= ~(flag);                               \
160                 }                                                       \
161         } while (0)
162
163         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
164         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
165         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
166
167         DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
168         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
169                 DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
170                               btrfs_raid_array[i].raid_name);
171 #undef DESCRIBE_FLAG
172
173         if (flags) {
174                 ret = snprintf(bp, size_bp, "0x%llx|", flags);
175                 size_bp -= ret;
176         }
177
178         if (size_bp < size_buf)
179                 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
180
181         /*
182          * The text is trimmed, it's up to the caller to provide sufficiently
183          * large buffer
184          */
185 out_overflow:;
186 }
187
188 static int init_first_rw_device(struct btrfs_trans_handle *trans,
189                                 struct btrfs_fs_info *fs_info);
190 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
191 static void __btrfs_reset_dev_stats(struct btrfs_device *dev);
192 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
193 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
194 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
195                              enum btrfs_map_op op,
196                              u64 logical, u64 *length,
197                              struct btrfs_bio **bbio_ret,
198                              int mirror_num, int need_raid_map);
199
200 /*
201  * Device locking
202  * ==============
203  *
204  * There are several mutexes that protect manipulation of devices and low-level
205  * structures like chunks but not block groups, extents or files
206  *
207  * uuid_mutex (global lock)
208  * ------------------------
209  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
210  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
211  * device) or requested by the device= mount option
212  *
213  * the mutex can be very coarse and can cover long-running operations
214  *
215  * protects: updates to fs_devices counters like missing devices, rw devices,
216  * seeding, structure cloning, opening/closing devices at mount/umount time
217  *
218  * global::fs_devs - add, remove, updates to the global list
219  *
220  * does not protect: manipulation of the fs_devices::devices list!
221  *
222  * btrfs_device::name - renames (write side), read is RCU
223  *
224  * fs_devices::device_list_mutex (per-fs, with RCU)
225  * ------------------------------------------------
226  * protects updates to fs_devices::devices, ie. adding and deleting
227  *
228  * simple list traversal with read-only actions can be done with RCU protection
229  *
230  * may be used to exclude some operations from running concurrently without any
231  * modifications to the list (see write_all_supers)
232  *
233  * balance_mutex
234  * -------------
235  * protects balance structures (status, state) and context accessed from
236  * several places (internally, ioctl)
237  *
238  * chunk_mutex
239  * -----------
240  * protects chunks, adding or removing during allocation, trim or when a new
241  * device is added/removed
242  *
243  * cleaner_mutex
244  * -------------
245  * a big lock that is held by the cleaner thread and prevents running subvolume
246  * cleaning together with relocation or delayed iputs
247  *
248  *
249  * Lock nesting
250  * ============
251  *
252  * uuid_mutex
253  *   volume_mutex
254  *     device_list_mutex
255  *       chunk_mutex
256  *     balance_mutex
257  *
258  *
259  * Exclusive operations, BTRFS_FS_EXCL_OP
260  * ======================================
261  *
262  * Maintains the exclusivity of the following operations that apply to the
263  * whole filesystem and cannot run in parallel.
264  *
265  * - Balance (*)
266  * - Device add
267  * - Device remove
268  * - Device replace (*)
269  * - Resize
270  *
271  * The device operations (as above) can be in one of the following states:
272  *
273  * - Running state
274  * - Paused state
275  * - Completed state
276  *
277  * Only device operations marked with (*) can go into the Paused state for the
278  * following reasons:
279  *
280  * - ioctl (only Balance can be Paused through ioctl)
281  * - filesystem remounted as read-only
282  * - filesystem unmounted and mounted as read-only
283  * - system power-cycle and filesystem mounted as read-only
284  * - filesystem or device errors leading to forced read-only
285  *
286  * BTRFS_FS_EXCL_OP flag is set and cleared using atomic operations.
287  * During the course of Paused state, the BTRFS_FS_EXCL_OP remains set.
288  * A device operation in Paused or Running state can be canceled or resumed
289  * either by ioctl (Balance only) or when remounted as read-write.
290  * BTRFS_FS_EXCL_OP flag is cleared when the device operation is canceled or
291  * completed.
292  */
293
294 DEFINE_MUTEX(uuid_mutex);
295 static LIST_HEAD(fs_uuids);
296 struct list_head *btrfs_get_fs_uuids(void)
297 {
298         return &fs_uuids;
299 }
300
301 /*
302  * alloc_fs_devices - allocate struct btrfs_fs_devices
303  * @fsid:               if not NULL, copy the UUID to fs_devices::fsid
304  * @metadata_fsid:      if not NULL, copy the UUID to fs_devices::metadata_fsid
305  *
306  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
307  * The returned struct is not linked onto any lists and can be destroyed with
308  * kfree() right away.
309  */
310 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
311                                                  const u8 *metadata_fsid)
312 {
313         struct btrfs_fs_devices *fs_devs;
314
315         fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
316         if (!fs_devs)
317                 return ERR_PTR(-ENOMEM);
318
319         mutex_init(&fs_devs->device_list_mutex);
320
321         INIT_LIST_HEAD(&fs_devs->devices);
322         INIT_LIST_HEAD(&fs_devs->resized_devices);
323         INIT_LIST_HEAD(&fs_devs->alloc_list);
324         INIT_LIST_HEAD(&fs_devs->fs_list);
325         if (fsid)
326                 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
327
328         if (metadata_fsid)
329                 memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
330         else if (fsid)
331                 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
332
333         return fs_devs;
334 }
335
336 void btrfs_free_device(struct btrfs_device *device)
337 {
338         rcu_string_free(device->name);
339         bio_put(device->flush_bio);
340         kfree(device);
341 }
342
343 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
344 {
345         struct btrfs_device *device;
346         WARN_ON(fs_devices->opened);
347         while (!list_empty(&fs_devices->devices)) {
348                 device = list_entry(fs_devices->devices.next,
349                                     struct btrfs_device, dev_list);
350                 list_del(&device->dev_list);
351                 btrfs_free_device(device);
352         }
353         kfree(fs_devices);
354 }
355
356 static void btrfs_kobject_uevent(struct block_device *bdev,
357                                  enum kobject_action action)
358 {
359         int ret;
360
361         ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
362         if (ret)
363                 pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
364                         action,
365                         kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
366                         &disk_to_dev(bdev->bd_disk)->kobj);
367 }
368
369 void __exit btrfs_cleanup_fs_uuids(void)
370 {
371         struct btrfs_fs_devices *fs_devices;
372
373         while (!list_empty(&fs_uuids)) {
374                 fs_devices = list_entry(fs_uuids.next,
375                                         struct btrfs_fs_devices, fs_list);
376                 list_del(&fs_devices->fs_list);
377                 free_fs_devices(fs_devices);
378         }
379 }
380
381 /*
382  * Returns a pointer to a new btrfs_device on success; ERR_PTR() on error.
383  * Returned struct is not linked onto any lists and must be destroyed using
384  * btrfs_free_device.
385  */
386 static struct btrfs_device *__alloc_device(void)
387 {
388         struct btrfs_device *dev;
389
390         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
391         if (!dev)
392                 return ERR_PTR(-ENOMEM);
393
394         /*
395          * Preallocate a bio that's always going to be used for flushing device
396          * barriers and matches the device lifespan
397          */
398         dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
399         if (!dev->flush_bio) {
400                 kfree(dev);
401                 return ERR_PTR(-ENOMEM);
402         }
403
404         INIT_LIST_HEAD(&dev->dev_list);
405         INIT_LIST_HEAD(&dev->dev_alloc_list);
406         INIT_LIST_HEAD(&dev->resized_list);
407
408         spin_lock_init(&dev->io_lock);
409
410         atomic_set(&dev->reada_in_flight, 0);
411         atomic_set(&dev->dev_stats_ccnt, 0);
412         btrfs_device_data_ordered_init(dev);
413         INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
414         INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
415
416         return dev;
417 }
418
419 static noinline struct btrfs_fs_devices *find_fsid(
420                 const u8 *fsid, const u8 *metadata_fsid)
421 {
422         struct btrfs_fs_devices *fs_devices;
423
424         ASSERT(fsid);
425
426         if (metadata_fsid) {
427                 /*
428                  * Handle scanned device having completed its fsid change but
429                  * belonging to a fs_devices that was created by first scanning
430                  * a device which didn't have its fsid/metadata_uuid changed
431                  * at all and the CHANGING_FSID_V2 flag set.
432                  */
433                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
434                         if (fs_devices->fsid_change &&
435                             memcmp(metadata_fsid, fs_devices->fsid,
436                                    BTRFS_FSID_SIZE) == 0 &&
437                             memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
438                                    BTRFS_FSID_SIZE) == 0) {
439                                 return fs_devices;
440                         }
441                 }
442                 /*
443                  * Handle scanned device having completed its fsid change but
444                  * belonging to a fs_devices that was created by a device that
445                  * has an outdated pair of fsid/metadata_uuid and
446                  * CHANGING_FSID_V2 flag set.
447                  */
448                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
449                         if (fs_devices->fsid_change &&
450                             memcmp(fs_devices->metadata_uuid,
451                                    fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
452                             memcmp(metadata_fsid, fs_devices->metadata_uuid,
453                                    BTRFS_FSID_SIZE) == 0) {
454                                 return fs_devices;
455                         }
456                 }
457         }
458
459         /* Handle non-split brain cases */
460         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
461                 if (metadata_fsid) {
462                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
463                             && memcmp(metadata_fsid, fs_devices->metadata_uuid,
464                                       BTRFS_FSID_SIZE) == 0)
465                                 return fs_devices;
466                 } else {
467                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
468                                 return fs_devices;
469                 }
470         }
471         return NULL;
472 }
473
474 static int
475 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
476                       int flush, struct block_device **bdev,
477                       struct buffer_head **bh)
478 {
479         int ret;
480
481         *bdev = blkdev_get_by_path(device_path, flags, holder);
482
483         if (IS_ERR(*bdev)) {
484                 ret = PTR_ERR(*bdev);
485                 goto error;
486         }
487
488         if (flush)
489                 filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
490         ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
491         if (ret) {
492                 blkdev_put(*bdev, flags);
493                 goto error;
494         }
495         invalidate_bdev(*bdev);
496         *bh = btrfs_read_dev_super(*bdev);
497         if (IS_ERR(*bh)) {
498                 ret = PTR_ERR(*bh);
499                 blkdev_put(*bdev, flags);
500                 goto error;
501         }
502
503         return 0;
504
505 error:
506         *bdev = NULL;
507         *bh = NULL;
508         return ret;
509 }
510
511 static void requeue_list(struct btrfs_pending_bios *pending_bios,
512                         struct bio *head, struct bio *tail)
513 {
514
515         struct bio *old_head;
516
517         old_head = pending_bios->head;
518         pending_bios->head = head;
519         if (pending_bios->tail)
520                 tail->bi_next = old_head;
521         else
522                 pending_bios->tail = tail;
523 }
524
525 /*
526  * we try to collect pending bios for a device so we don't get a large
527  * number of procs sending bios down to the same device.  This greatly
528  * improves the schedulers ability to collect and merge the bios.
529  *
530  * But, it also turns into a long list of bios to process and that is sure
531  * to eventually make the worker thread block.  The solution here is to
532  * make some progress and then put this work struct back at the end of
533  * the list if the block device is congested.  This way, multiple devices
534  * can make progress from a single worker thread.
535  */
536 static noinline void run_scheduled_bios(struct btrfs_device *device)
537 {
538         struct btrfs_fs_info *fs_info = device->fs_info;
539         struct bio *pending;
540         struct backing_dev_info *bdi;
541         struct btrfs_pending_bios *pending_bios;
542         struct bio *tail;
543         struct bio *cur;
544         int again = 0;
545         unsigned long num_run;
546         unsigned long batch_run = 0;
547         unsigned long last_waited = 0;
548         int force_reg = 0;
549         int sync_pending = 0;
550         struct blk_plug plug;
551
552         /*
553          * this function runs all the bios we've collected for
554          * a particular device.  We don't want to wander off to
555          * another device without first sending all of these down.
556          * So, setup a plug here and finish it off before we return
557          */
558         blk_start_plug(&plug);
559
560         bdi = device->bdev->bd_bdi;
561
562 loop:
563         spin_lock(&device->io_lock);
564
565 loop_lock:
566         num_run = 0;
567
568         /* take all the bios off the list at once and process them
569          * later on (without the lock held).  But, remember the
570          * tail and other pointers so the bios can be properly reinserted
571          * into the list if we hit congestion
572          */
573         if (!force_reg && device->pending_sync_bios.head) {
574                 pending_bios = &device->pending_sync_bios;
575                 force_reg = 1;
576         } else {
577                 pending_bios = &device->pending_bios;
578                 force_reg = 0;
579         }
580
581         pending = pending_bios->head;
582         tail = pending_bios->tail;
583         WARN_ON(pending && !tail);
584
585         /*
586          * if pending was null this time around, no bios need processing
587          * at all and we can stop.  Otherwise it'll loop back up again
588          * and do an additional check so no bios are missed.
589          *
590          * device->running_pending is used to synchronize with the
591          * schedule_bio code.
592          */
593         if (device->pending_sync_bios.head == NULL &&
594             device->pending_bios.head == NULL) {
595                 again = 0;
596                 device->running_pending = 0;
597         } else {
598                 again = 1;
599                 device->running_pending = 1;
600         }
601
602         pending_bios->head = NULL;
603         pending_bios->tail = NULL;
604
605         spin_unlock(&device->io_lock);
606
607         while (pending) {
608
609                 rmb();
610                 /* we want to work on both lists, but do more bios on the
611                  * sync list than the regular list
612                  */
613                 if ((num_run > 32 &&
614                     pending_bios != &device->pending_sync_bios &&
615                     device->pending_sync_bios.head) ||
616                    (num_run > 64 && pending_bios == &device->pending_sync_bios &&
617                     device->pending_bios.head)) {
618                         spin_lock(&device->io_lock);
619                         requeue_list(pending_bios, pending, tail);
620                         goto loop_lock;
621                 }
622
623                 cur = pending;
624                 pending = pending->bi_next;
625                 cur->bi_next = NULL;
626
627                 BUG_ON(atomic_read(&cur->__bi_cnt) == 0);
628
629                 /*
630                  * if we're doing the sync list, record that our
631                  * plug has some sync requests on it
632                  *
633                  * If we're doing the regular list and there are
634                  * sync requests sitting around, unplug before
635                  * we add more
636                  */
637                 if (pending_bios == &device->pending_sync_bios) {
638                         sync_pending = 1;
639                 } else if (sync_pending) {
640                         blk_finish_plug(&plug);
641                         blk_start_plug(&plug);
642                         sync_pending = 0;
643                 }
644
645                 btrfsic_submit_bio(cur);
646                 num_run++;
647                 batch_run++;
648
649                 cond_resched();
650
651                 /*
652                  * we made progress, there is more work to do and the bdi
653                  * is now congested.  Back off and let other work structs
654                  * run instead
655                  */
656                 if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
657                     fs_info->fs_devices->open_devices > 1) {
658                         struct io_context *ioc;
659
660                         ioc = current->io_context;
661
662                         /*
663                          * the main goal here is that we don't want to
664                          * block if we're going to be able to submit
665                          * more requests without blocking.
666                          *
667                          * This code does two great things, it pokes into
668                          * the elevator code from a filesystem _and_
669                          * it makes assumptions about how batching works.
670                          */
671                         if (ioc && ioc->nr_batch_requests > 0 &&
672                             time_before(jiffies, ioc->last_waited + HZ/50UL) &&
673                             (last_waited == 0 ||
674                              ioc->last_waited == last_waited)) {
675                                 /*
676                                  * we want to go through our batch of
677                                  * requests and stop.  So, we copy out
678                                  * the ioc->last_waited time and test
679                                  * against it before looping
680                                  */
681                                 last_waited = ioc->last_waited;
682                                 cond_resched();
683                                 continue;
684                         }
685                         spin_lock(&device->io_lock);
686                         requeue_list(pending_bios, pending, tail);
687                         device->running_pending = 1;
688
689                         spin_unlock(&device->io_lock);
690                         btrfs_queue_work(fs_info->submit_workers,
691                                          &device->work);
692                         goto done;
693                 }
694         }
695
696         cond_resched();
697         if (again)
698                 goto loop;
699
700         spin_lock(&device->io_lock);
701         if (device->pending_bios.head || device->pending_sync_bios.head)
702                 goto loop_lock;
703         spin_unlock(&device->io_lock);
704
705 done:
706         blk_finish_plug(&plug);
707 }
708
709 static void pending_bios_fn(struct btrfs_work *work)
710 {
711         struct btrfs_device *device;
712
713         device = container_of(work, struct btrfs_device, work);
714         run_scheduled_bios(device);
715 }
716
717 static bool device_path_matched(const char *path, struct btrfs_device *device)
718 {
719         int found;
720
721         rcu_read_lock();
722         found = strcmp(rcu_str_deref(device->name), path);
723         rcu_read_unlock();
724
725         return found == 0;
726 }
727
728 /*
729  *  Search and remove all stale (devices which are not mounted) devices.
730  *  When both inputs are NULL, it will search and release all stale devices.
731  *  path:       Optional. When provided will it release all unmounted devices
732  *              matching this path only.
733  *  skip_dev:   Optional. Will skip this device when searching for the stale
734  *              devices.
735  *  Return:     0 for success or if @path is NULL.
736  *              -EBUSY if @path is a mounted device.
737  *              -ENOENT if @path does not match any device in the list.
738  */
739 static int btrfs_free_stale_devices(const char *path,
740                                      struct btrfs_device *skip_device)
741 {
742         struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
743         struct btrfs_device *device, *tmp_device;
744         int ret = 0;
745
746         if (path)
747                 ret = -ENOENT;
748
749         list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
750
751                 mutex_lock(&fs_devices->device_list_mutex);
752                 list_for_each_entry_safe(device, tmp_device,
753                                          &fs_devices->devices, dev_list) {
754                         if (skip_device && skip_device == device)
755                                 continue;
756                         if (path && !device->name)
757                                 continue;
758                         if (path && !device_path_matched(path, device))
759                                 continue;
760                         if (fs_devices->opened) {
761                                 /* for an already deleted device return 0 */
762                                 if (path && ret != 0)
763                                         ret = -EBUSY;
764                                 break;
765                         }
766
767                         /* delete the stale device */
768                         fs_devices->num_devices--;
769                         list_del(&device->dev_list);
770                         btrfs_free_device(device);
771
772                         ret = 0;
773                         if (fs_devices->num_devices == 0)
774                                 break;
775                 }
776                 mutex_unlock(&fs_devices->device_list_mutex);
777
778                 if (fs_devices->num_devices == 0) {
779                         btrfs_sysfs_remove_fsid(fs_devices);
780                         list_del(&fs_devices->fs_list);
781                         free_fs_devices(fs_devices);
782                 }
783         }
784
785         return ret;
786 }
787
788 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
789                         struct btrfs_device *device, fmode_t flags,
790                         void *holder)
791 {
792         struct request_queue *q;
793         struct block_device *bdev;
794         struct buffer_head *bh;
795         struct btrfs_super_block *disk_super;
796         u64 devid;
797         int ret;
798
799         if (device->bdev)
800                 return -EINVAL;
801         if (!device->name)
802                 return -EINVAL;
803
804         ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
805                                     &bdev, &bh);
806         if (ret)
807                 return ret;
808
809         disk_super = (struct btrfs_super_block *)bh->b_data;
810         devid = btrfs_stack_device_id(&disk_super->dev_item);
811         if (devid != device->devid)
812                 goto error_brelse;
813
814         if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
815                 goto error_brelse;
816
817         device->generation = btrfs_super_generation(disk_super);
818
819         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
820                 if (btrfs_super_incompat_flags(disk_super) &
821                     BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
822                         pr_err(
823                 "BTRFS: Invalid seeding and uuid-changed device detected\n");
824                         goto error_brelse;
825                 }
826
827                 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
828                 fs_devices->seeding = 1;
829         } else {
830                 if (bdev_read_only(bdev))
831                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
832                 else
833                         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
834         }
835
836         q = bdev_get_queue(bdev);
837         if (!blk_queue_nonrot(q))
838                 fs_devices->rotating = 1;
839
840         device->bdev = bdev;
841         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
842         device->mode = flags;
843
844         fs_devices->open_devices++;
845         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
846             device->devid != BTRFS_DEV_REPLACE_DEVID) {
847                 fs_devices->rw_devices++;
848                 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
849         }
850         brelse(bh);
851
852         return 0;
853
854 error_brelse:
855         brelse(bh);
856         blkdev_put(bdev, flags);
857
858         return -EINVAL;
859 }
860
861 /*
862  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
863  * being created with a disk that has already completed its fsid change.
864  */
865 static struct btrfs_fs_devices *find_fsid_inprogress(
866                                         struct btrfs_super_block *disk_super)
867 {
868         struct btrfs_fs_devices *fs_devices;
869
870         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
871                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
872                            BTRFS_FSID_SIZE) != 0 &&
873                     memcmp(fs_devices->metadata_uuid, disk_super->fsid,
874                            BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
875                         return fs_devices;
876                 }
877         }
878
879         return NULL;
880 }
881
882
883 static struct btrfs_fs_devices *find_fsid_changed(
884                                         struct btrfs_super_block *disk_super)
885 {
886         struct btrfs_fs_devices *fs_devices;
887
888         /*
889          * Handles the case where scanned device is part of an fs that had
890          * multiple successful changes of FSID but curently device didn't
891          * observe it. Meaning our fsid will be different than theirs.
892          */
893         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
894                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
895                            BTRFS_FSID_SIZE) != 0 &&
896                     memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
897                            BTRFS_FSID_SIZE) == 0 &&
898                     memcmp(fs_devices->fsid, disk_super->fsid,
899                            BTRFS_FSID_SIZE) != 0) {
900                         return fs_devices;
901                 }
902         }
903
904         return NULL;
905 }
906 /*
907  * Add new device to list of registered devices
908  *
909  * Returns:
910  * device pointer which was just added or updated when successful
911  * error pointer when failed
912  */
913 static noinline struct btrfs_device *device_list_add(const char *path,
914                            struct btrfs_super_block *disk_super,
915                            bool *new_device_added)
916 {
917         struct btrfs_device *device;
918         struct btrfs_fs_devices *fs_devices = NULL;
919         struct rcu_string *name;
920         u64 found_transid = btrfs_super_generation(disk_super);
921         u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
922         bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
923                 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
924         bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
925                                         BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
926
927         if (fsid_change_in_progress) {
928                 if (!has_metadata_uuid) {
929                         /*
930                          * When we have an image which has CHANGING_FSID_V2 set
931                          * it might belong to either a filesystem which has
932                          * disks with completed fsid change or it might belong
933                          * to fs with no UUID changes in effect, handle both.
934                          */
935                         fs_devices = find_fsid_inprogress(disk_super);
936                         if (!fs_devices)
937                                 fs_devices = find_fsid(disk_super->fsid, NULL);
938                 } else {
939                         fs_devices = find_fsid_changed(disk_super);
940                 }
941         } else if (has_metadata_uuid) {
942                 fs_devices = find_fsid(disk_super->fsid,
943                                        disk_super->metadata_uuid);
944         } else {
945                 fs_devices = find_fsid(disk_super->fsid, NULL);
946         }
947
948
949         if (!fs_devices) {
950                 if (has_metadata_uuid)
951                         fs_devices = alloc_fs_devices(disk_super->fsid,
952                                                       disk_super->metadata_uuid);
953                 else
954                         fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
955
956                 if (IS_ERR(fs_devices))
957                         return ERR_CAST(fs_devices);
958
959                 fs_devices->fsid_change = fsid_change_in_progress;
960
961                 mutex_lock(&fs_devices->device_list_mutex);
962                 list_add(&fs_devices->fs_list, &fs_uuids);
963
964                 device = NULL;
965         } else {
966                 mutex_lock(&fs_devices->device_list_mutex);
967                 device = btrfs_find_device(fs_devices, devid,
968                                 disk_super->dev_item.uuid, NULL, false);
969
970                 /*
971                  * If this disk has been pulled into an fs devices created by
972                  * a device which had the CHANGING_FSID_V2 flag then replace the
973                  * metadata_uuid/fsid values of the fs_devices.
974                  */
975                 if (has_metadata_uuid && fs_devices->fsid_change &&
976                     found_transid > fs_devices->latest_generation) {
977                         memcpy(fs_devices->fsid, disk_super->fsid,
978                                         BTRFS_FSID_SIZE);
979                         memcpy(fs_devices->metadata_uuid,
980                                         disk_super->metadata_uuid, BTRFS_FSID_SIZE);
981
982                         fs_devices->fsid_change = false;
983                 }
984         }
985
986         if (!device) {
987                 if (fs_devices->opened) {
988                         mutex_unlock(&fs_devices->device_list_mutex);
989                         return ERR_PTR(-EBUSY);
990                 }
991
992                 device = btrfs_alloc_device(NULL, &devid,
993                                             disk_super->dev_item.uuid);
994                 if (IS_ERR(device)) {
995                         mutex_unlock(&fs_devices->device_list_mutex);
996                         /* we can safely leave the fs_devices entry around */
997                         return device;
998                 }
999
1000                 name = rcu_string_strdup(path, GFP_NOFS);
1001                 if (!name) {
1002                         btrfs_free_device(device);
1003                         mutex_unlock(&fs_devices->device_list_mutex);
1004                         return ERR_PTR(-ENOMEM);
1005                 }
1006                 rcu_assign_pointer(device->name, name);
1007
1008                 list_add_rcu(&device->dev_list, &fs_devices->devices);
1009                 fs_devices->num_devices++;
1010
1011                 device->fs_devices = fs_devices;
1012                 *new_device_added = true;
1013
1014                 if (disk_super->label[0])
1015                         pr_info("BTRFS: device label %s devid %llu transid %llu %s\n",
1016                                 disk_super->label, devid, found_transid, path);
1017                 else
1018                         pr_info("BTRFS: device fsid %pU devid %llu transid %llu %s\n",
1019                                 disk_super->fsid, devid, found_transid, path);
1020
1021         } else if (!device->name || strcmp(device->name->str, path)) {
1022                 /*
1023                  * When FS is already mounted.
1024                  * 1. If you are here and if the device->name is NULL that
1025                  *    means this device was missing at time of FS mount.
1026                  * 2. If you are here and if the device->name is different
1027                  *    from 'path' that means either
1028                  *      a. The same device disappeared and reappeared with
1029                  *         different name. or
1030                  *      b. The missing-disk-which-was-replaced, has
1031                  *         reappeared now.
1032                  *
1033                  * We must allow 1 and 2a above. But 2b would be a spurious
1034                  * and unintentional.
1035                  *
1036                  * Further in case of 1 and 2a above, the disk at 'path'
1037                  * would have missed some transaction when it was away and
1038                  * in case of 2a the stale bdev has to be updated as well.
1039                  * 2b must not be allowed at all time.
1040                  */
1041
1042                 /*
1043                  * For now, we do allow update to btrfs_fs_device through the
1044                  * btrfs dev scan cli after FS has been mounted.  We're still
1045                  * tracking a problem where systems fail mount by subvolume id
1046                  * when we reject replacement on a mounted FS.
1047                  */
1048                 if (!fs_devices->opened && found_transid < device->generation) {
1049                         /*
1050                          * That is if the FS is _not_ mounted and if you
1051                          * are here, that means there is more than one
1052                          * disk with same uuid and devid.We keep the one
1053                          * with larger generation number or the last-in if
1054                          * generation are equal.
1055                          */
1056                         mutex_unlock(&fs_devices->device_list_mutex);
1057                         return ERR_PTR(-EEXIST);
1058                 }
1059
1060                 /*
1061                  * We are going to replace the device path for a given devid,
1062                  * make sure it's the same device if the device is mounted
1063                  */
1064                 if (device->bdev) {
1065                         struct block_device *path_bdev;
1066
1067                         path_bdev = lookup_bdev(path);
1068                         if (IS_ERR(path_bdev)) {
1069                                 mutex_unlock(&fs_devices->device_list_mutex);
1070                                 return ERR_CAST(path_bdev);
1071                         }
1072
1073                         if (device->bdev != path_bdev) {
1074                                 bdput(path_bdev);
1075                                 mutex_unlock(&fs_devices->device_list_mutex);
1076                                 btrfs_warn_in_rcu(device->fs_info,
1077                         "duplicate device fsid:devid for %pU:%llu old:%s new:%s",
1078                                         disk_super->fsid, devid,
1079                                         rcu_str_deref(device->name), path);
1080                                 return ERR_PTR(-EEXIST);
1081                         }
1082                         bdput(path_bdev);
1083                         btrfs_info_in_rcu(device->fs_info,
1084                                 "device fsid %pU devid %llu moved old:%s new:%s",
1085                                 disk_super->fsid, devid,
1086                                 rcu_str_deref(device->name), path);
1087                 }
1088
1089                 name = rcu_string_strdup(path, GFP_NOFS);
1090                 if (!name) {
1091                         mutex_unlock(&fs_devices->device_list_mutex);
1092                         return ERR_PTR(-ENOMEM);
1093                 }
1094                 rcu_string_free(device->name);
1095                 rcu_assign_pointer(device->name, name);
1096                 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1097                         fs_devices->missing_devices--;
1098                         clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1099                 }
1100         }
1101
1102         /*
1103          * Unmount does not free the btrfs_device struct but would zero
1104          * generation along with most of the other members. So just update
1105          * it back. We need it to pick the disk with largest generation
1106          * (as above).
1107          */
1108         if (!fs_devices->opened) {
1109                 device->generation = found_transid;
1110                 fs_devices->latest_generation = max_t(u64, found_transid,
1111                                                 fs_devices->latest_generation);
1112         }
1113
1114         fs_devices->total_devices = btrfs_super_num_devices(disk_super);
1115
1116         mutex_unlock(&fs_devices->device_list_mutex);
1117         return device;
1118 }
1119
1120 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
1121 {
1122         struct btrfs_fs_devices *fs_devices;
1123         struct btrfs_device *device;
1124         struct btrfs_device *orig_dev;
1125
1126         fs_devices = alloc_fs_devices(orig->fsid, NULL);
1127         if (IS_ERR(fs_devices))
1128                 return fs_devices;
1129
1130         mutex_lock(&orig->device_list_mutex);
1131         fs_devices->total_devices = orig->total_devices;
1132
1133         list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1134                 struct rcu_string *name;
1135
1136                 device = btrfs_alloc_device(NULL, &orig_dev->devid,
1137                                             orig_dev->uuid);
1138                 if (IS_ERR(device))
1139                         goto error;
1140
1141                 /*
1142                  * This is ok to do without rcu read locked because we hold the
1143                  * uuid mutex so nothing we touch in here is going to disappear.
1144                  */
1145                 if (orig_dev->name) {
1146                         name = rcu_string_strdup(orig_dev->name->str,
1147                                         GFP_KERNEL);
1148                         if (!name) {
1149                                 btrfs_free_device(device);
1150                                 goto error;
1151                         }
1152                         rcu_assign_pointer(device->name, name);
1153                 }
1154
1155                 list_add(&device->dev_list, &fs_devices->devices);
1156                 device->fs_devices = fs_devices;
1157                 fs_devices->num_devices++;
1158         }
1159         mutex_unlock(&orig->device_list_mutex);
1160         return fs_devices;
1161 error:
1162         mutex_unlock(&orig->device_list_mutex);
1163         free_fs_devices(fs_devices);
1164         return ERR_PTR(-ENOMEM);
1165 }
1166
1167 /*
1168  * After we have read the system tree and know devids belonging to
1169  * this filesystem, remove the device which does not belong there.
1170  */
1171 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
1172 {
1173         struct btrfs_device *device, *next;
1174         struct btrfs_device *latest_dev = NULL;
1175
1176         mutex_lock(&uuid_mutex);
1177 again:
1178         /* This is the initialized path, it is safe to release the devices. */
1179         list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1180                 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
1181                                                         &device->dev_state)) {
1182                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1183                              &device->dev_state) &&
1184                              (!latest_dev ||
1185                               device->generation > latest_dev->generation)) {
1186                                 latest_dev = device;
1187                         }
1188                         continue;
1189                 }
1190
1191                 if (device->devid == BTRFS_DEV_REPLACE_DEVID) {
1192                         /*
1193                          * In the first step, keep the device which has
1194                          * the correct fsid and the devid that is used
1195                          * for the dev_replace procedure.
1196                          * In the second step, the dev_replace state is
1197                          * read from the device tree and it is known
1198                          * whether the procedure is really active or
1199                          * not, which means whether this device is
1200                          * used or whether it should be removed.
1201                          */
1202                         if (step == 0 || test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1203                                                   &device->dev_state)) {
1204                                 continue;
1205                         }
1206                 }
1207                 if (device->bdev) {
1208                         blkdev_put(device->bdev, device->mode);
1209                         device->bdev = NULL;
1210                         fs_devices->open_devices--;
1211                 }
1212                 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1213                         list_del_init(&device->dev_alloc_list);
1214                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1215                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1216                                       &device->dev_state))
1217                                 fs_devices->rw_devices--;
1218                 }
1219                 list_del_init(&device->dev_list);
1220                 fs_devices->num_devices--;
1221                 btrfs_free_device(device);
1222         }
1223
1224         if (fs_devices->seed) {
1225                 fs_devices = fs_devices->seed;
1226                 goto again;
1227         }
1228
1229         fs_devices->latest_bdev = latest_dev->bdev;
1230
1231         mutex_unlock(&uuid_mutex);
1232 }
1233
1234 static void free_device_rcu(struct rcu_head *head)
1235 {
1236         struct btrfs_device *device;
1237
1238         device = container_of(head, struct btrfs_device, rcu);
1239         btrfs_free_device(device);
1240 }
1241
1242 static void btrfs_close_bdev(struct btrfs_device *device)
1243 {
1244         if (!device->bdev)
1245                 return;
1246
1247         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1248                 sync_blockdev(device->bdev);
1249                 invalidate_bdev(device->bdev);
1250         }
1251
1252         blkdev_put(device->bdev, device->mode);
1253 }
1254
1255 static void btrfs_close_one_device(struct btrfs_device *device)
1256 {
1257         struct btrfs_fs_devices *fs_devices = device->fs_devices;
1258         struct btrfs_device *new_device;
1259         struct rcu_string *name;
1260
1261         if (device->bdev)
1262                 fs_devices->open_devices--;
1263
1264         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1265             device->devid != BTRFS_DEV_REPLACE_DEVID) {
1266                 list_del_init(&device->dev_alloc_list);
1267                 fs_devices->rw_devices--;
1268         }
1269
1270         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
1271                 fs_devices->missing_devices--;
1272
1273         btrfs_close_bdev(device);
1274
1275         new_device = btrfs_alloc_device(NULL, &device->devid,
1276                                         device->uuid);
1277         BUG_ON(IS_ERR(new_device)); /* -ENOMEM */
1278
1279         /* Safe because we are under uuid_mutex */
1280         if (device->name) {
1281                 name = rcu_string_strdup(device->name->str, GFP_NOFS);
1282                 BUG_ON(!name); /* -ENOMEM */
1283                 rcu_assign_pointer(new_device->name, name);
1284         }
1285
1286         list_replace_rcu(&device->dev_list, &new_device->dev_list);
1287         new_device->fs_devices = device->fs_devices;
1288
1289         call_rcu(&device->rcu, free_device_rcu);
1290 }
1291
1292 static int close_fs_devices(struct btrfs_fs_devices *fs_devices)
1293 {
1294         struct btrfs_device *device, *tmp;
1295
1296         if (--fs_devices->opened > 0)
1297                 return 0;
1298
1299         mutex_lock(&fs_devices->device_list_mutex);
1300         list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) {
1301                 btrfs_close_one_device(device);
1302         }
1303         mutex_unlock(&fs_devices->device_list_mutex);
1304
1305         WARN_ON(fs_devices->open_devices);
1306         WARN_ON(fs_devices->rw_devices);
1307         fs_devices->opened = 0;
1308         fs_devices->seeding = 0;
1309
1310         return 0;
1311 }
1312
1313 int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1314 {
1315         struct btrfs_fs_devices *seed_devices = NULL;
1316         int ret;
1317
1318         mutex_lock(&uuid_mutex);
1319         ret = close_fs_devices(fs_devices);
1320         if (!fs_devices->opened) {
1321                 seed_devices = fs_devices->seed;
1322                 fs_devices->seed = NULL;
1323         }
1324         mutex_unlock(&uuid_mutex);
1325
1326         while (seed_devices) {
1327                 fs_devices = seed_devices;
1328                 seed_devices = fs_devices->seed;
1329                 close_fs_devices(fs_devices);
1330                 free_fs_devices(fs_devices);
1331         }
1332         return ret;
1333 }
1334
1335 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1336                                 fmode_t flags, void *holder)
1337 {
1338         struct btrfs_device *device;
1339         struct btrfs_device *latest_dev = NULL;
1340         int ret = 0;
1341
1342         flags |= FMODE_EXCL;
1343
1344         list_for_each_entry(device, &fs_devices->devices, dev_list) {
1345                 /* Just open everything we can; ignore failures here */
1346                 if (btrfs_open_one_device(fs_devices, device, flags, holder))
1347                         continue;
1348
1349                 if (!latest_dev ||
1350                     device->generation > latest_dev->generation)
1351                         latest_dev = device;
1352         }
1353         if (fs_devices->open_devices == 0) {
1354                 ret = -EINVAL;
1355                 goto out;
1356         }
1357         fs_devices->opened = 1;
1358         fs_devices->latest_bdev = latest_dev->bdev;
1359         fs_devices->total_rw_bytes = 0;
1360 out:
1361         return ret;
1362 }
1363
1364 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
1365 {
1366         struct btrfs_device *dev1, *dev2;
1367
1368         dev1 = list_entry(a, struct btrfs_device, dev_list);
1369         dev2 = list_entry(b, struct btrfs_device, dev_list);
1370
1371         if (dev1->devid < dev2->devid)
1372                 return -1;
1373         else if (dev1->devid > dev2->devid)
1374                 return 1;
1375         return 0;
1376 }
1377
1378 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1379                        fmode_t flags, void *holder)
1380 {
1381         int ret;
1382
1383         lockdep_assert_held(&uuid_mutex);
1384
1385         mutex_lock(&fs_devices->device_list_mutex);
1386         if (fs_devices->opened) {
1387                 fs_devices->opened++;
1388                 ret = 0;
1389         } else {
1390                 list_sort(NULL, &fs_devices->devices, devid_cmp);
1391                 ret = open_fs_devices(fs_devices, flags, holder);
1392         }
1393         mutex_unlock(&fs_devices->device_list_mutex);
1394
1395         return ret;
1396 }
1397
1398 static void btrfs_release_disk_super(struct page *page)
1399 {
1400         kunmap(page);
1401         put_page(page);
1402 }
1403
1404 static int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr,
1405                                  struct page **page,
1406                                  struct btrfs_super_block **disk_super)
1407 {
1408         void *p;
1409         pgoff_t index;
1410
1411         /* make sure our super fits in the device */
1412         if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1413                 return 1;
1414
1415         /* make sure our super fits in the page */
1416         if (sizeof(**disk_super) > PAGE_SIZE)
1417                 return 1;
1418
1419         /* make sure our super doesn't straddle pages on disk */
1420         index = bytenr >> PAGE_SHIFT;
1421         if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index)
1422                 return 1;
1423
1424         /* pull in the page with our super */
1425         *page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
1426                                    index, GFP_KERNEL);
1427
1428         if (IS_ERR_OR_NULL(*page))
1429                 return 1;
1430
1431         p = kmap(*page);
1432
1433         /* align our pointer to the offset of the super block */
1434         *disk_super = p + offset_in_page(bytenr);
1435
1436         if (btrfs_super_bytenr(*disk_super) != bytenr ||
1437             btrfs_super_magic(*disk_super) != BTRFS_MAGIC) {
1438                 btrfs_release_disk_super(*page);
1439                 return 1;
1440         }
1441
1442         if ((*disk_super)->label[0] &&
1443                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1])
1444                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0';
1445
1446         return 0;
1447 }
1448
1449 int btrfs_forget_devices(const char *path)
1450 {
1451         int ret;
1452
1453         mutex_lock(&uuid_mutex);
1454         ret = btrfs_free_stale_devices(strlen(path) ? path : NULL, NULL);
1455         mutex_unlock(&uuid_mutex);
1456
1457         return ret;
1458 }
1459
1460 /*
1461  * Look for a btrfs signature on a device. This may be called out of the mount path
1462  * and we are not allowed to call set_blocksize during the scan. The superblock
1463  * is read via pagecache
1464  */
1465 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1466                                            void *holder)
1467 {
1468         struct btrfs_super_block *disk_super;
1469         bool new_device_added = false;
1470         struct btrfs_device *device = NULL;
1471         struct block_device *bdev;
1472         struct page *page;
1473         u64 bytenr;
1474
1475         lockdep_assert_held(&uuid_mutex);
1476
1477         /*
1478          * we would like to check all the supers, but that would make
1479          * a btrfs mount succeed after a mkfs from a different FS.
1480          * So, we need to add a special mount option to scan for
1481          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1482          */
1483         bytenr = btrfs_sb_offset(0);
1484         flags |= FMODE_EXCL;
1485
1486         bdev = blkdev_get_by_path(path, flags, holder);
1487         if (IS_ERR(bdev))
1488                 return ERR_CAST(bdev);
1489
1490         if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super)) {
1491                 device = ERR_PTR(-EINVAL);
1492                 goto error_bdev_put;
1493         }
1494
1495         device = device_list_add(path, disk_super, &new_device_added);
1496         if (!IS_ERR(device)) {
1497                 if (new_device_added)
1498                         btrfs_free_stale_devices(path, device);
1499         }
1500
1501         btrfs_release_disk_super(page);
1502
1503 error_bdev_put:
1504         blkdev_put(bdev, flags);
1505
1506         return device;
1507 }
1508
1509 static int contains_pending_extent(struct btrfs_transaction *transaction,
1510                                    struct btrfs_device *device,
1511                                    u64 *start, u64 len)
1512 {
1513         struct btrfs_fs_info *fs_info = device->fs_info;
1514         struct extent_map *em;
1515         struct list_head *search_list = &fs_info->pinned_chunks;
1516         int ret = 0;
1517         u64 physical_start = *start;
1518
1519         if (transaction)
1520                 search_list = &transaction->pending_chunks;
1521 again:
1522         list_for_each_entry(em, search_list, list) {
1523                 struct map_lookup *map;
1524                 int i;
1525
1526                 map = em->map_lookup;
1527                 for (i = 0; i < map->num_stripes; i++) {
1528                         u64 end;
1529
1530                         if (map->stripes[i].dev != device)
1531                                 continue;
1532                         if (map->stripes[i].physical >= physical_start + len ||
1533                             map->stripes[i].physical + em->orig_block_len <=
1534                             physical_start)
1535                                 continue;
1536                         /*
1537                          * Make sure that while processing the pinned list we do
1538                          * not override our *start with a lower value, because
1539                          * we can have pinned chunks that fall within this
1540                          * device hole and that have lower physical addresses
1541                          * than the pending chunks we processed before. If we
1542                          * do not take this special care we can end up getting
1543                          * 2 pending chunks that start at the same physical
1544                          * device offsets because the end offset of a pinned
1545                          * chunk can be equal to the start offset of some
1546                          * pending chunk.
1547                          */
1548                         end = map->stripes[i].physical + em->orig_block_len;
1549                         if (end > *start) {
1550                                 *start = end;
1551                                 ret = 1;
1552                         }
1553                 }
1554         }
1555         if (search_list != &fs_info->pinned_chunks) {
1556                 search_list = &fs_info->pinned_chunks;
1557                 goto again;
1558         }
1559
1560         return ret;
1561 }
1562
1563
1564 /*
1565  * find_free_dev_extent_start - find free space in the specified device
1566  * @device:       the device which we search the free space in
1567  * @num_bytes:    the size of the free space that we need
1568  * @search_start: the position from which to begin the search
1569  * @start:        store the start of the free space.
1570  * @len:          the size of the free space. that we find, or the size
1571  *                of the max free space if we don't find suitable free space
1572  *
1573  * this uses a pretty simple search, the expectation is that it is
1574  * called very infrequently and that a given device has a small number
1575  * of extents
1576  *
1577  * @start is used to store the start of the free space if we find. But if we
1578  * don't find suitable free space, it will be used to store the start position
1579  * of the max free space.
1580  *
1581  * @len is used to store the size of the free space that we find.
1582  * But if we don't find suitable free space, it is used to store the size of
1583  * the max free space.
1584  */
1585 int find_free_dev_extent_start(struct btrfs_transaction *transaction,
1586                                struct btrfs_device *device, u64 num_bytes,
1587                                u64 search_start, u64 *start, u64 *len)
1588 {
1589         struct btrfs_fs_info *fs_info = device->fs_info;
1590         struct btrfs_root *root = fs_info->dev_root;
1591         struct btrfs_key key;
1592         struct btrfs_dev_extent *dev_extent;
1593         struct btrfs_path *path;
1594         u64 hole_size;
1595         u64 max_hole_start;
1596         u64 max_hole_size;
1597         u64 extent_end;
1598         u64 search_end = device->total_bytes;
1599         int ret;
1600         int slot;
1601         struct extent_buffer *l;
1602
1603         /*
1604          * We don't want to overwrite the superblock on the drive nor any area
1605          * used by the boot loader (grub for example), so we make sure to start
1606          * at an offset of at least 1MB.
1607          */
1608         search_start = max_t(u64, search_start, SZ_1M);
1609
1610         path = btrfs_alloc_path();
1611         if (!path)
1612                 return -ENOMEM;
1613
1614         max_hole_start = search_start;
1615         max_hole_size = 0;
1616
1617 again:
1618         if (search_start >= search_end ||
1619                 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1620                 ret = -ENOSPC;
1621                 goto out;
1622         }
1623
1624         path->reada = READA_FORWARD;
1625         path->search_commit_root = 1;
1626         path->skip_locking = 1;
1627
1628         key.objectid = device->devid;
1629         key.offset = search_start;
1630         key.type = BTRFS_DEV_EXTENT_KEY;
1631
1632         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1633         if (ret < 0)
1634                 goto out;
1635         if (ret > 0) {
1636                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1637                 if (ret < 0)
1638                         goto out;
1639         }
1640
1641         while (1) {
1642                 l = path->nodes[0];
1643                 slot = path->slots[0];
1644                 if (slot >= btrfs_header_nritems(l)) {
1645                         ret = btrfs_next_leaf(root, path);
1646                         if (ret == 0)
1647                                 continue;
1648                         if (ret < 0)
1649                                 goto out;
1650
1651                         break;
1652                 }
1653                 btrfs_item_key_to_cpu(l, &key, slot);
1654
1655                 if (key.objectid < device->devid)
1656                         goto next;
1657
1658                 if (key.objectid > device->devid)
1659                         break;
1660
1661                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1662                         goto next;
1663
1664                 if (key.offset > search_start) {
1665                         hole_size = key.offset - search_start;
1666
1667                         /*
1668                          * Have to check before we set max_hole_start, otherwise
1669                          * we could end up sending back this offset anyway.
1670                          */
1671                         if (contains_pending_extent(transaction, device,
1672                                                     &search_start,
1673                                                     hole_size)) {
1674                                 if (key.offset >= search_start) {
1675                                         hole_size = key.offset - search_start;
1676                                 } else {
1677                                         WARN_ON_ONCE(1);
1678                                         hole_size = 0;
1679                                 }
1680                         }
1681
1682                         if (hole_size > max_hole_size) {
1683                                 max_hole_start = search_start;
1684                                 max_hole_size = hole_size;
1685                         }
1686
1687                         /*
1688                          * If this free space is greater than which we need,
1689                          * it must be the max free space that we have found
1690                          * until now, so max_hole_start must point to the start
1691                          * of this free space and the length of this free space
1692                          * is stored in max_hole_size. Thus, we return
1693                          * max_hole_start and max_hole_size and go back to the
1694                          * caller.
1695                          */
1696                         if (hole_size >= num_bytes) {
1697                                 ret = 0;
1698                                 goto out;
1699                         }
1700                 }
1701
1702                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1703                 extent_end = key.offset + btrfs_dev_extent_length(l,
1704                                                                   dev_extent);
1705                 if (extent_end > search_start)
1706                         search_start = extent_end;
1707 next:
1708                 path->slots[0]++;
1709                 cond_resched();
1710         }
1711
1712         /*
1713          * At this point, search_start should be the end of
1714          * allocated dev extents, and when shrinking the device,
1715          * search_end may be smaller than search_start.
1716          */
1717         if (search_end > search_start) {
1718                 hole_size = search_end - search_start;
1719
1720                 if (contains_pending_extent(transaction, device, &search_start,
1721                                             hole_size)) {
1722                         btrfs_release_path(path);
1723                         goto again;
1724                 }
1725
1726                 if (hole_size > max_hole_size) {
1727                         max_hole_start = search_start;
1728                         max_hole_size = hole_size;
1729                 }
1730         }
1731
1732         /* See above. */
1733         if (max_hole_size < num_bytes)
1734                 ret = -ENOSPC;
1735         else
1736                 ret = 0;
1737
1738 out:
1739         btrfs_free_path(path);
1740         *start = max_hole_start;
1741         if (len)
1742                 *len = max_hole_size;
1743         return ret;
1744 }
1745
1746 int find_free_dev_extent(struct btrfs_trans_handle *trans,
1747                          struct btrfs_device *device, u64 num_bytes,
1748                          u64 *start, u64 *len)
1749 {
1750         /* FIXME use last free of some kind */
1751         return find_free_dev_extent_start(trans->transaction, device,
1752                                           num_bytes, 0, start, len);
1753 }
1754
1755 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1756                           struct btrfs_device *device,
1757                           u64 start, u64 *dev_extent_len)
1758 {
1759         struct btrfs_fs_info *fs_info = device->fs_info;
1760         struct btrfs_root *root = fs_info->dev_root;
1761         int ret;
1762         struct btrfs_path *path;
1763         struct btrfs_key key;
1764         struct btrfs_key found_key;
1765         struct extent_buffer *leaf = NULL;
1766         struct btrfs_dev_extent *extent = NULL;
1767
1768         path = btrfs_alloc_path();
1769         if (!path)
1770                 return -ENOMEM;
1771
1772         key.objectid = device->devid;
1773         key.offset = start;
1774         key.type = BTRFS_DEV_EXTENT_KEY;
1775 again:
1776         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1777         if (ret > 0) {
1778                 ret = btrfs_previous_item(root, path, key.objectid,
1779                                           BTRFS_DEV_EXTENT_KEY);
1780                 if (ret)
1781                         goto out;
1782                 leaf = path->nodes[0];
1783                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1784                 extent = btrfs_item_ptr(leaf, path->slots[0],
1785                                         struct btrfs_dev_extent);
1786                 BUG_ON(found_key.offset > start || found_key.offset +
1787                        btrfs_dev_extent_length(leaf, extent) < start);
1788                 key = found_key;
1789                 btrfs_release_path(path);
1790                 goto again;
1791         } else if (ret == 0) {
1792                 leaf = path->nodes[0];
1793                 extent = btrfs_item_ptr(leaf, path->slots[0],
1794                                         struct btrfs_dev_extent);
1795         } else {
1796                 btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1797                 goto out;
1798         }
1799
1800         *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1801
1802         ret = btrfs_del_item(trans, root, path);
1803         if (ret) {
1804                 btrfs_handle_fs_error(fs_info, ret,
1805                                       "Failed to remove dev extent item");
1806         } else {
1807                 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1808         }
1809 out:
1810         btrfs_free_path(path);
1811         return ret;
1812 }
1813
1814 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1815                                   struct btrfs_device *device,
1816                                   u64 chunk_offset, u64 start, u64 num_bytes)
1817 {
1818         int ret;
1819         struct btrfs_path *path;
1820         struct btrfs_fs_info *fs_info = device->fs_info;
1821         struct btrfs_root *root = fs_info->dev_root;
1822         struct btrfs_dev_extent *extent;
1823         struct extent_buffer *leaf;
1824         struct btrfs_key key;
1825
1826         WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state));
1827         WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1828         path = btrfs_alloc_path();
1829         if (!path)
1830                 return -ENOMEM;
1831
1832         key.objectid = device->devid;
1833         key.offset = start;
1834         key.type = BTRFS_DEV_EXTENT_KEY;
1835         ret = btrfs_insert_empty_item(trans, root, path, &key,
1836                                       sizeof(*extent));
1837         if (ret)
1838                 goto out;
1839
1840         leaf = path->nodes[0];
1841         extent = btrfs_item_ptr(leaf, path->slots[0],
1842                                 struct btrfs_dev_extent);
1843         btrfs_set_dev_extent_chunk_tree(leaf, extent,
1844                                         BTRFS_CHUNK_TREE_OBJECTID);
1845         btrfs_set_dev_extent_chunk_objectid(leaf, extent,
1846                                             BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1847         btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1848
1849         btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1850         btrfs_mark_buffer_dirty(leaf);
1851 out:
1852         btrfs_free_path(path);
1853         return ret;
1854 }
1855
1856 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1857 {
1858         struct extent_map_tree *em_tree;
1859         struct extent_map *em;
1860         struct rb_node *n;
1861         u64 ret = 0;
1862
1863         em_tree = &fs_info->mapping_tree.map_tree;
1864         read_lock(&em_tree->lock);
1865         n = rb_last(&em_tree->map.rb_root);
1866         if (n) {
1867                 em = rb_entry(n, struct extent_map, rb_node);
1868                 ret = em->start + em->len;
1869         }
1870         read_unlock(&em_tree->lock);
1871
1872         return ret;
1873 }
1874
1875 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1876                                     u64 *devid_ret)
1877 {
1878         int ret;
1879         struct btrfs_key key;
1880         struct btrfs_key found_key;
1881         struct btrfs_path *path;
1882
1883         path = btrfs_alloc_path();
1884         if (!path)
1885                 return -ENOMEM;
1886
1887         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1888         key.type = BTRFS_DEV_ITEM_KEY;
1889         key.offset = (u64)-1;
1890
1891         ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1892         if (ret < 0)
1893                 goto error;
1894
1895         BUG_ON(ret == 0); /* Corruption */
1896
1897         ret = btrfs_previous_item(fs_info->chunk_root, path,
1898                                   BTRFS_DEV_ITEMS_OBJECTID,
1899                                   BTRFS_DEV_ITEM_KEY);
1900         if (ret) {
1901                 *devid_ret = 1;
1902         } else {
1903                 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1904                                       path->slots[0]);
1905                 *devid_ret = found_key.offset + 1;
1906         }
1907         ret = 0;
1908 error:
1909         btrfs_free_path(path);
1910         return ret;
1911 }
1912
1913 /*
1914  * the device information is stored in the chunk root
1915  * the btrfs_device struct should be fully filled in
1916  */
1917 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1918                             struct btrfs_device *device)
1919 {
1920         int ret;
1921         struct btrfs_path *path;
1922         struct btrfs_dev_item *dev_item;
1923         struct extent_buffer *leaf;
1924         struct btrfs_key key;
1925         unsigned long ptr;
1926
1927         path = btrfs_alloc_path();
1928         if (!path)
1929                 return -ENOMEM;
1930
1931         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1932         key.type = BTRFS_DEV_ITEM_KEY;
1933         key.offset = device->devid;
1934
1935         ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1936                                       &key, sizeof(*dev_item));
1937         if (ret)
1938                 goto out;
1939
1940         leaf = path->nodes[0];
1941         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1942
1943         btrfs_set_device_id(leaf, dev_item, device->devid);
1944         btrfs_set_device_generation(leaf, dev_item, 0);
1945         btrfs_set_device_type(leaf, dev_item, device->type);
1946         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1947         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1948         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1949         btrfs_set_device_total_bytes(leaf, dev_item,
1950                                      btrfs_device_get_disk_total_bytes(device));
1951         btrfs_set_device_bytes_used(leaf, dev_item,
1952                                     btrfs_device_get_bytes_used(device));
1953         btrfs_set_device_group(leaf, dev_item, 0);
1954         btrfs_set_device_seek_speed(leaf, dev_item, 0);
1955         btrfs_set_device_bandwidth(leaf, dev_item, 0);
1956         btrfs_set_device_start_offset(leaf, dev_item, 0);
1957
1958         ptr = btrfs_device_uuid(dev_item);
1959         write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1960         ptr = btrfs_device_fsid(dev_item);
1961         write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1962                             ptr, BTRFS_FSID_SIZE);
1963         btrfs_mark_buffer_dirty(leaf);
1964
1965         ret = 0;
1966 out:
1967         btrfs_free_path(path);
1968         return ret;
1969 }
1970
1971 /*
1972  * Function to update ctime/mtime for a given device path.
1973  * Mainly used for ctime/mtime based probe like libblkid.
1974  */
1975 static void update_dev_time(const char *path_name)
1976 {
1977         struct file *filp;
1978
1979         filp = filp_open(path_name, O_RDWR, 0);
1980         if (IS_ERR(filp))
1981                 return;
1982         file_update_time(filp);
1983         filp_close(filp, NULL);
1984 }
1985
1986 static int btrfs_rm_dev_item(struct btrfs_fs_info *fs_info,
1987                              struct btrfs_device *device)
1988 {
1989         struct btrfs_root *root = fs_info->chunk_root;
1990         int ret;
1991         struct btrfs_path *path;
1992         struct btrfs_key key;
1993         struct btrfs_trans_handle *trans;
1994
1995         path = btrfs_alloc_path();
1996         if (!path)
1997                 return -ENOMEM;
1998
1999         trans = btrfs_start_transaction(root, 0);
2000         if (IS_ERR(trans)) {
2001                 btrfs_free_path(path);
2002                 return PTR_ERR(trans);
2003         }
2004         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2005         key.type = BTRFS_DEV_ITEM_KEY;
2006         key.offset = device->devid;
2007
2008         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2009         if (ret) {
2010                 if (ret > 0)
2011                         ret = -ENOENT;
2012                 btrfs_abort_transaction(trans, ret);
2013                 btrfs_end_transaction(trans);
2014                 goto out;
2015         }
2016
2017         ret = btrfs_del_item(trans, root, path);
2018         if (ret) {
2019                 btrfs_abort_transaction(trans, ret);
2020                 btrfs_end_transaction(trans);
2021         }
2022
2023 out:
2024         btrfs_free_path(path);
2025         if (!ret)
2026                 ret = btrfs_commit_transaction(trans);
2027         return ret;
2028 }
2029
2030 /*
2031  * Verify that @num_devices satisfies the RAID profile constraints in the whole
2032  * filesystem. It's up to the caller to adjust that number regarding eg. device
2033  * replace.
2034  */
2035 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
2036                 u64 num_devices)
2037 {
2038         u64 all_avail;
2039         unsigned seq;
2040         int i;
2041
2042         do {
2043                 seq = read_seqbegin(&fs_info->profiles_lock);
2044
2045                 all_avail = fs_info->avail_data_alloc_bits |
2046                             fs_info->avail_system_alloc_bits |
2047                             fs_info->avail_metadata_alloc_bits;
2048         } while (read_seqretry(&fs_info->profiles_lock, seq));
2049
2050         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2051                 if (!(all_avail & btrfs_raid_array[i].bg_flag))
2052                         continue;
2053
2054                 if (num_devices < btrfs_raid_array[i].devs_min) {
2055                         int ret = btrfs_raid_array[i].mindev_error;
2056
2057                         if (ret)
2058                                 return ret;
2059                 }
2060         }
2061
2062         return 0;
2063 }
2064
2065 static struct btrfs_device * btrfs_find_next_active_device(
2066                 struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2067 {
2068         struct btrfs_device *next_device;
2069
2070         list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2071                 if (next_device != device &&
2072                     !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2073                     && next_device->bdev)
2074                         return next_device;
2075         }
2076
2077         return NULL;
2078 }
2079
2080 /*
2081  * Helper function to check if the given device is part of s_bdev / latest_bdev
2082  * and replace it with the provided or the next active device, in the context
2083  * where this function called, there should be always be another device (or
2084  * this_dev) which is active.
2085  */
2086 void btrfs_assign_next_active_device(struct btrfs_device *device,
2087                                      struct btrfs_device *this_dev)
2088 {
2089         struct btrfs_fs_info *fs_info = device->fs_info;
2090         struct btrfs_device *next_device;
2091
2092         if (this_dev)
2093                 next_device = this_dev;
2094         else
2095                 next_device = btrfs_find_next_active_device(fs_info->fs_devices,
2096                                                                 device);
2097         ASSERT(next_device);
2098
2099         if (fs_info->sb->s_bdev &&
2100                         (fs_info->sb->s_bdev == device->bdev))
2101                 fs_info->sb->s_bdev = next_device->bdev;
2102
2103         if (fs_info->fs_devices->latest_bdev == device->bdev)
2104                 fs_info->fs_devices->latest_bdev = next_device->bdev;
2105 }
2106
2107 /*
2108  * Return btrfs_fs_devices::num_devices excluding the device that's being
2109  * currently replaced.
2110  */
2111 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2112 {
2113         u64 num_devices = fs_info->fs_devices->num_devices;
2114
2115         down_read(&fs_info->dev_replace.rwsem);
2116         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
2117                 ASSERT(num_devices > 1);
2118                 num_devices--;
2119         }
2120         up_read(&fs_info->dev_replace.rwsem);
2121
2122         return num_devices;
2123 }
2124
2125 int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
2126                 u64 devid)
2127 {
2128         struct btrfs_device *device;
2129         struct btrfs_fs_devices *cur_devices;
2130         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2131         u64 num_devices;
2132         int ret = 0;
2133
2134         mutex_lock(&uuid_mutex);
2135
2136         num_devices = btrfs_num_devices(fs_info);
2137
2138         ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2139         if (ret)
2140                 goto out;
2141
2142         device = btrfs_find_device_by_devspec(fs_info, devid, device_path);
2143
2144         if (IS_ERR(device)) {
2145                 if (PTR_ERR(device) == -ENOENT &&
2146                     strcmp(device_path, "missing") == 0)
2147                         ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2148                 else
2149                         ret = PTR_ERR(device);
2150                 goto out;
2151         }
2152
2153         if (btrfs_pinned_by_swapfile(fs_info, device)) {
2154                 btrfs_warn_in_rcu(fs_info,
2155                   "cannot remove device %s (devid %llu) due to active swapfile",
2156                                   rcu_str_deref(device->name), device->devid);
2157                 ret = -ETXTBSY;
2158                 goto out;
2159         }
2160
2161         if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2162                 ret = BTRFS_ERROR_DEV_TGT_REPLACE;
2163                 goto out;
2164         }
2165
2166         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2167             fs_info->fs_devices->rw_devices == 1) {
2168                 ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
2169                 goto out;
2170         }
2171
2172         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2173                 mutex_lock(&fs_info->chunk_mutex);
2174                 list_del_init(&device->dev_alloc_list);
2175                 device->fs_devices->rw_devices--;
2176                 mutex_unlock(&fs_info->chunk_mutex);
2177         }
2178
2179         mutex_unlock(&uuid_mutex);
2180         ret = btrfs_shrink_device(device, 0);
2181         mutex_lock(&uuid_mutex);
2182         if (ret)
2183                 goto error_undo;
2184
2185         /*
2186          * TODO: the superblock still includes this device in its num_devices
2187          * counter although write_all_supers() is not locked out. This
2188          * could give a filesystem state which requires a degraded mount.
2189          */
2190         ret = btrfs_rm_dev_item(fs_info, device);
2191         if (ret)
2192                 goto error_undo;
2193
2194         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2195         btrfs_scrub_cancel_dev(fs_info, device);
2196
2197         /*
2198          * the device list mutex makes sure that we don't change
2199          * the device list while someone else is writing out all
2200          * the device supers. Whoever is writing all supers, should
2201          * lock the device list mutex before getting the number of
2202          * devices in the super block (super_copy). Conversely,
2203          * whoever updates the number of devices in the super block
2204          * (super_copy) should hold the device list mutex.
2205          */
2206
2207         /*
2208          * In normal cases the cur_devices == fs_devices. But in case
2209          * of deleting a seed device, the cur_devices should point to
2210          * its own fs_devices listed under the fs_devices->seed.
2211          */
2212         cur_devices = device->fs_devices;
2213         mutex_lock(&fs_devices->device_list_mutex);
2214         list_del_rcu(&device->dev_list);
2215
2216         cur_devices->num_devices--;
2217         cur_devices->total_devices--;
2218         /* Update total_devices of the parent fs_devices if it's seed */
2219         if (cur_devices != fs_devices)
2220                 fs_devices->total_devices--;
2221
2222         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2223                 cur_devices->missing_devices--;
2224
2225         btrfs_assign_next_active_device(device, NULL);
2226
2227         if (device->bdev) {
2228                 cur_devices->open_devices--;
2229                 /* remove sysfs entry */
2230                 btrfs_sysfs_rm_device_link(fs_devices, device);
2231         }
2232
2233         num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2234         btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2235         mutex_unlock(&fs_devices->device_list_mutex);
2236
2237         /*
2238          * at this point, the device is zero sized and detached from
2239          * the devices list.  All that's left is to zero out the old
2240          * supers and free the device.
2241          */
2242         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2243                 btrfs_scratch_superblocks(device->bdev, device->name->str);
2244
2245         btrfs_close_bdev(device);
2246         call_rcu(&device->rcu, free_device_rcu);
2247
2248         if (cur_devices->open_devices == 0) {
2249                 while (fs_devices) {
2250                         if (fs_devices->seed == cur_devices) {
2251                                 fs_devices->seed = cur_devices->seed;
2252                                 break;
2253                         }
2254                         fs_devices = fs_devices->seed;
2255                 }
2256                 cur_devices->seed = NULL;
2257                 close_fs_devices(cur_devices);
2258                 free_fs_devices(cur_devices);
2259         }
2260
2261 out:
2262         mutex_unlock(&uuid_mutex);
2263         return ret;
2264
2265 error_undo:
2266         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2267                 mutex_lock(&fs_info->chunk_mutex);
2268                 list_add(&device->dev_alloc_list,
2269                          &fs_devices->alloc_list);
2270                 device->fs_devices->rw_devices++;
2271                 mutex_unlock(&fs_info->chunk_mutex);
2272         }
2273         goto out;
2274 }
2275
2276 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2277 {
2278         struct btrfs_fs_devices *fs_devices;
2279
2280         lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2281
2282         /*
2283          * in case of fs with no seed, srcdev->fs_devices will point
2284          * to fs_devices of fs_info. However when the dev being replaced is
2285          * a seed dev it will point to the seed's local fs_devices. In short
2286          * srcdev will have its correct fs_devices in both the cases.
2287          */
2288         fs_devices = srcdev->fs_devices;
2289
2290         list_del_rcu(&srcdev->dev_list);
2291         list_del(&srcdev->dev_alloc_list);
2292         fs_devices->num_devices--;
2293         if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2294                 fs_devices->missing_devices--;
2295
2296         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2297                 fs_devices->rw_devices--;
2298
2299         if (srcdev->bdev)
2300                 fs_devices->open_devices--;
2301 }
2302
2303 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_fs_info *fs_info,
2304                                       struct btrfs_device *srcdev)
2305 {
2306         struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2307
2308         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) {
2309                 /* zero out the old super if it is writable */
2310                 btrfs_scratch_superblocks(srcdev->bdev, srcdev->name->str);
2311         }
2312
2313         btrfs_close_bdev(srcdev);
2314         call_rcu(&srcdev->rcu, free_device_rcu);
2315
2316         /* if this is no devs we rather delete the fs_devices */
2317         if (!fs_devices->num_devices) {
2318                 struct btrfs_fs_devices *tmp_fs_devices;
2319
2320                 /*
2321                  * On a mounted FS, num_devices can't be zero unless it's a
2322                  * seed. In case of a seed device being replaced, the replace
2323                  * target added to the sprout FS, so there will be no more
2324                  * device left under the seed FS.
2325                  */
2326                 ASSERT(fs_devices->seeding);
2327
2328                 tmp_fs_devices = fs_info->fs_devices;
2329                 while (tmp_fs_devices) {
2330                         if (tmp_fs_devices->seed == fs_devices) {
2331                                 tmp_fs_devices->seed = fs_devices->seed;
2332                                 break;
2333                         }
2334                         tmp_fs_devices = tmp_fs_devices->seed;
2335                 }
2336                 fs_devices->seed = NULL;
2337                 close_fs_devices(fs_devices);
2338                 free_fs_devices(fs_devices);
2339         }
2340 }
2341
2342 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2343 {
2344         struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2345
2346         WARN_ON(!tgtdev);
2347         mutex_lock(&fs_devices->device_list_mutex);
2348
2349         btrfs_sysfs_rm_device_link(fs_devices, tgtdev);
2350
2351         if (tgtdev->bdev)
2352                 fs_devices->open_devices--;
2353
2354         fs_devices->num_devices--;
2355
2356         btrfs_assign_next_active_device(tgtdev, NULL);
2357
2358         list_del_rcu(&tgtdev->dev_list);
2359
2360         mutex_unlock(&fs_devices->device_list_mutex);
2361
2362         /*
2363          * The update_dev_time() with in btrfs_scratch_superblocks()
2364          * may lead to a call to btrfs_show_devname() which will try
2365          * to hold device_list_mutex. And here this device
2366          * is already out of device list, so we don't have to hold
2367          * the device_list_mutex lock.
2368          */
2369         btrfs_scratch_superblocks(tgtdev->bdev, tgtdev->name->str);
2370
2371         btrfs_close_bdev(tgtdev);
2372         call_rcu(&tgtdev->rcu, free_device_rcu);
2373 }
2374
2375 static struct btrfs_device *btrfs_find_device_by_path(
2376                 struct btrfs_fs_info *fs_info, const char *device_path)
2377 {
2378         int ret = 0;
2379         struct btrfs_super_block *disk_super;
2380         u64 devid;
2381         u8 *dev_uuid;
2382         struct block_device *bdev;
2383         struct buffer_head *bh;
2384         struct btrfs_device *device;
2385
2386         ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2387                                     fs_info->bdev_holder, 0, &bdev, &bh);
2388         if (ret)
2389                 return ERR_PTR(ret);
2390         disk_super = (struct btrfs_super_block *)bh->b_data;
2391         devid = btrfs_stack_device_id(&disk_super->dev_item);
2392         dev_uuid = disk_super->dev_item.uuid;
2393         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2394                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2395                                            disk_super->metadata_uuid, true);
2396         else
2397                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2398                                            disk_super->fsid, true);
2399
2400         brelse(bh);
2401         if (!device)
2402                 device = ERR_PTR(-ENOENT);
2403         blkdev_put(bdev, FMODE_READ);
2404         return device;
2405 }
2406
2407 /*
2408  * Lookup a device given by device id, or the path if the id is 0.
2409  */
2410 struct btrfs_device *btrfs_find_device_by_devspec(
2411                 struct btrfs_fs_info *fs_info, u64 devid,
2412                 const char *device_path)
2413 {
2414         struct btrfs_device *device;
2415
2416         if (devid) {
2417                 device = btrfs_find_device(fs_info->fs_devices, devid, NULL,
2418                                            NULL, true);
2419                 if (!device)
2420                         return ERR_PTR(-ENOENT);
2421                 return device;
2422         }
2423
2424         if (!device_path || !device_path[0])
2425                 return ERR_PTR(-EINVAL);
2426
2427         if (strcmp(device_path, "missing") == 0) {
2428                 /* Find first missing device */
2429                 list_for_each_entry(device, &fs_info->fs_devices->devices,
2430                                     dev_list) {
2431                         if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
2432                                      &device->dev_state) && !device->bdev)
2433                                 return device;
2434                 }
2435                 return ERR_PTR(-ENOENT);
2436         }
2437
2438         return btrfs_find_device_by_path(fs_info, device_path);
2439 }
2440
2441 /*
2442  * does all the dirty work required for changing file system's UUID.
2443  */
2444 static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
2445 {
2446         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2447         struct btrfs_fs_devices *old_devices;
2448         struct btrfs_fs_devices *seed_devices;
2449         struct btrfs_super_block *disk_super = fs_info->super_copy;
2450         struct btrfs_device *device;
2451         u64 super_flags;
2452
2453         lockdep_assert_held(&uuid_mutex);
2454         if (!fs_devices->seeding)
2455                 return -EINVAL;
2456
2457         seed_devices = alloc_fs_devices(NULL, NULL);
2458         if (IS_ERR(seed_devices))
2459                 return PTR_ERR(seed_devices);
2460
2461         old_devices = clone_fs_devices(fs_devices);
2462         if (IS_ERR(old_devices)) {
2463                 kfree(seed_devices);
2464                 return PTR_ERR(old_devices);
2465         }
2466
2467         list_add(&old_devices->fs_list, &fs_uuids);
2468
2469         memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2470         seed_devices->opened = 1;
2471         INIT_LIST_HEAD(&seed_devices->devices);
2472         INIT_LIST_HEAD(&seed_devices->alloc_list);
2473         mutex_init(&seed_devices->device_list_mutex);
2474
2475         mutex_lock(&fs_devices->device_list_mutex);
2476         list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2477                               synchronize_rcu);
2478         list_for_each_entry(device, &seed_devices->devices, dev_list)
2479                 device->fs_devices = seed_devices;
2480
2481         mutex_lock(&fs_info->chunk_mutex);
2482         list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
2483         mutex_unlock(&fs_info->chunk_mutex);
2484
2485         fs_devices->seeding = 0;
2486         fs_devices->num_devices = 0;
2487         fs_devices->open_devices = 0;
2488         fs_devices->missing_devices = 0;
2489         fs_devices->rotating = 0;
2490         fs_devices->seed = seed_devices;
2491
2492         generate_random_uuid(fs_devices->fsid);
2493         memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2494         memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2495         mutex_unlock(&fs_devices->device_list_mutex);
2496
2497         super_flags = btrfs_super_flags(disk_super) &
2498                       ~BTRFS_SUPER_FLAG_SEEDING;
2499         btrfs_set_super_flags(disk_super, super_flags);
2500
2501         return 0;
2502 }
2503
2504 /*
2505  * Store the expected generation for seed devices in device items.
2506  */
2507 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans,
2508                                struct btrfs_fs_info *fs_info)
2509 {
2510         struct btrfs_root *root = fs_info->chunk_root;
2511         struct btrfs_path *path;
2512         struct extent_buffer *leaf;
2513         struct btrfs_dev_item *dev_item;
2514         struct btrfs_device *device;
2515         struct btrfs_key key;
2516         u8 fs_uuid[BTRFS_FSID_SIZE];
2517         u8 dev_uuid[BTRFS_UUID_SIZE];
2518         u64 devid;
2519         int ret;
2520
2521         path = btrfs_alloc_path();
2522         if (!path)
2523                 return -ENOMEM;
2524
2525         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2526         key.offset = 0;
2527         key.type = BTRFS_DEV_ITEM_KEY;
2528
2529         while (1) {
2530                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2531                 if (ret < 0)
2532                         goto error;
2533
2534                 leaf = path->nodes[0];
2535 next_slot:
2536                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2537                         ret = btrfs_next_leaf(root, path);
2538                         if (ret > 0)
2539                                 break;
2540                         if (ret < 0)
2541                                 goto error;
2542                         leaf = path->nodes[0];
2543                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2544                         btrfs_release_path(path);
2545                         continue;
2546                 }
2547
2548                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2549                 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2550                     key.type != BTRFS_DEV_ITEM_KEY)
2551                         break;
2552
2553                 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2554                                           struct btrfs_dev_item);
2555                 devid = btrfs_device_id(leaf, dev_item);
2556                 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2557                                    BTRFS_UUID_SIZE);
2558                 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2559                                    BTRFS_FSID_SIZE);
2560                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2561                                            fs_uuid, true);
2562                 BUG_ON(!device); /* Logic error */
2563
2564                 if (device->fs_devices->seeding) {
2565                         btrfs_set_device_generation(leaf, dev_item,
2566                                                     device->generation);
2567                         btrfs_mark_buffer_dirty(leaf);
2568                 }
2569
2570                 path->slots[0]++;
2571                 goto next_slot;
2572         }
2573         ret = 0;
2574 error:
2575         btrfs_free_path(path);
2576         return ret;
2577 }
2578
2579 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2580 {
2581         struct btrfs_root *root = fs_info->dev_root;
2582         struct request_queue *q;
2583         struct btrfs_trans_handle *trans;
2584         struct btrfs_device *device;
2585         struct block_device *bdev;
2586         struct super_block *sb = fs_info->sb;
2587         struct rcu_string *name;
2588         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2589         u64 orig_super_total_bytes;
2590         u64 orig_super_num_devices;
2591         int seeding_dev = 0;
2592         int ret = 0;
2593         bool unlocked = false;
2594
2595         if (sb_rdonly(sb) && !fs_devices->seeding)
2596                 return -EROFS;
2597
2598         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2599                                   fs_info->bdev_holder);
2600         if (IS_ERR(bdev))
2601                 return PTR_ERR(bdev);
2602
2603         if (fs_devices->seeding) {
2604                 seeding_dev = 1;
2605                 down_write(&sb->s_umount);
2606                 mutex_lock(&uuid_mutex);
2607         }
2608
2609         filemap_write_and_wait(bdev->bd_inode->i_mapping);
2610
2611         mutex_lock(&fs_devices->device_list_mutex);
2612         list_for_each_entry(device, &fs_devices->devices, dev_list) {
2613                 if (device->bdev == bdev) {
2614                         ret = -EEXIST;
2615                         mutex_unlock(
2616                                 &fs_devices->device_list_mutex);
2617                         goto error;
2618                 }
2619         }
2620         mutex_unlock(&fs_devices->device_list_mutex);
2621
2622         device = btrfs_alloc_device(fs_info, NULL, NULL);
2623         if (IS_ERR(device)) {
2624                 /* we can safely leave the fs_devices entry around */
2625                 ret = PTR_ERR(device);
2626                 goto error;
2627         }
2628
2629         name = rcu_string_strdup(device_path, GFP_KERNEL);
2630         if (!name) {
2631                 ret = -ENOMEM;
2632                 goto error_free_device;
2633         }
2634         rcu_assign_pointer(device->name, name);
2635
2636         trans = btrfs_start_transaction(root, 0);
2637         if (IS_ERR(trans)) {
2638                 ret = PTR_ERR(trans);
2639                 goto error_free_device;
2640         }
2641
2642         q = bdev_get_queue(bdev);
2643         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2644         device->generation = trans->transid;
2645         device->io_width = fs_info->sectorsize;
2646         device->io_align = fs_info->sectorsize;
2647         device->sector_size = fs_info->sectorsize;
2648         device->total_bytes = round_down(i_size_read(bdev->bd_inode),
2649                                          fs_info->sectorsize);
2650         device->disk_total_bytes = device->total_bytes;
2651         device->commit_total_bytes = device->total_bytes;
2652         device->fs_info = fs_info;
2653         device->bdev = bdev;
2654         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2655         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2656         device->mode = FMODE_EXCL;
2657         device->dev_stats_valid = 1;
2658         set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2659
2660         if (seeding_dev) {
2661                 sb->s_flags &= ~SB_RDONLY;
2662                 ret = btrfs_prepare_sprout(fs_info);
2663                 if (ret) {
2664                         btrfs_abort_transaction(trans, ret);
2665                         goto error_trans;
2666                 }
2667         }
2668
2669         device->fs_devices = fs_devices;
2670
2671         mutex_lock(&fs_devices->device_list_mutex);
2672         mutex_lock(&fs_info->chunk_mutex);
2673         list_add_rcu(&device->dev_list, &fs_devices->devices);
2674         list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2675         fs_devices->num_devices++;
2676         fs_devices->open_devices++;
2677         fs_devices->rw_devices++;
2678         fs_devices->total_devices++;
2679         fs_devices->total_rw_bytes += device->total_bytes;
2680
2681         atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2682
2683         if (!blk_queue_nonrot(q))
2684                 fs_devices->rotating = 1;
2685
2686         orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2687         btrfs_set_super_total_bytes(fs_info->super_copy,
2688                 round_down(orig_super_total_bytes + device->total_bytes,
2689                            fs_info->sectorsize));
2690
2691         orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2692         btrfs_set_super_num_devices(fs_info->super_copy,
2693                                     orig_super_num_devices + 1);
2694
2695         /* add sysfs device entry */
2696         btrfs_sysfs_add_device_link(fs_devices, device);
2697
2698         /*
2699          * we've got more storage, clear any full flags on the space
2700          * infos
2701          */
2702         btrfs_clear_space_info_full(fs_info);
2703
2704         mutex_unlock(&fs_info->chunk_mutex);
2705         mutex_unlock(&fs_devices->device_list_mutex);
2706
2707         if (seeding_dev) {
2708                 mutex_lock(&fs_info->chunk_mutex);
2709                 ret = init_first_rw_device(trans, fs_info);
2710                 mutex_unlock(&fs_info->chunk_mutex);
2711                 if (ret) {
2712                         btrfs_abort_transaction(trans, ret);
2713                         goto error_sysfs;
2714                 }
2715         }
2716
2717         ret = btrfs_add_dev_item(trans, device);
2718         if (ret) {
2719                 btrfs_abort_transaction(trans, ret);
2720                 goto error_sysfs;
2721         }
2722
2723         if (seeding_dev) {
2724                 char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
2725
2726                 ret = btrfs_finish_sprout(trans, fs_info);
2727                 if (ret) {
2728                         btrfs_abort_transaction(trans, ret);
2729                         goto error_sysfs;
2730                 }
2731
2732                 /* Sprouting would change fsid of the mounted root,
2733                  * so rename the fsid on the sysfs
2734                  */
2735                 snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
2736                                                 fs_info->fs_devices->fsid);
2737                 if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf))
2738                         btrfs_warn(fs_info,
2739                                    "sysfs: failed to create fsid for sprout");
2740         }
2741
2742         ret = btrfs_commit_transaction(trans);
2743
2744         if (seeding_dev) {
2745                 mutex_unlock(&uuid_mutex);
2746                 up_write(&sb->s_umount);
2747                 unlocked = true;
2748
2749                 if (ret) /* transaction commit */
2750                         return ret;
2751
2752                 ret = btrfs_relocate_sys_chunks(fs_info);
2753                 if (ret < 0)
2754                         btrfs_handle_fs_error(fs_info, ret,
2755                                     "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2756                 trans = btrfs_attach_transaction(root);
2757                 if (IS_ERR(trans)) {
2758                         if (PTR_ERR(trans) == -ENOENT)
2759                                 return 0;
2760                         ret = PTR_ERR(trans);
2761                         trans = NULL;
2762                         goto error_sysfs;
2763                 }
2764                 ret = btrfs_commit_transaction(trans);
2765         }
2766
2767         /* Update ctime/mtime for libblkid */
2768         update_dev_time(device_path);
2769         return ret;
2770
2771 error_sysfs:
2772         btrfs_sysfs_rm_device_link(fs_devices, device);
2773         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2774         mutex_lock(&fs_info->chunk_mutex);
2775         list_del_rcu(&device->dev_list);
2776         list_del(&device->dev_alloc_list);
2777         fs_info->fs_devices->num_devices--;
2778         fs_info->fs_devices->open_devices--;
2779         fs_info->fs_devices->rw_devices--;
2780         fs_info->fs_devices->total_devices--;
2781         fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2782         atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2783         btrfs_set_super_total_bytes(fs_info->super_copy,
2784                                     orig_super_total_bytes);
2785         btrfs_set_super_num_devices(fs_info->super_copy,
2786                                     orig_super_num_devices);
2787         mutex_unlock(&fs_info->chunk_mutex);
2788         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2789 error_trans:
2790         if (seeding_dev)
2791                 sb->s_flags |= SB_RDONLY;
2792         if (trans)
2793                 btrfs_end_transaction(trans);
2794 error_free_device:
2795         btrfs_free_device(device);
2796 error:
2797         blkdev_put(bdev, FMODE_EXCL);
2798         if (seeding_dev && !unlocked) {
2799                 mutex_unlock(&uuid_mutex);
2800                 up_write(&sb->s_umount);
2801         }
2802         return ret;
2803 }
2804
2805 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2806                                         struct btrfs_device *device)
2807 {
2808         int ret;
2809         struct btrfs_path *path;
2810         struct btrfs_root *root = device->fs_info->chunk_root;
2811         struct btrfs_dev_item *dev_item;
2812         struct extent_buffer *leaf;
2813         struct btrfs_key key;
2814
2815         path = btrfs_alloc_path();
2816         if (!path)
2817                 return -ENOMEM;
2818
2819         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2820         key.type = BTRFS_DEV_ITEM_KEY;
2821         key.offset = device->devid;
2822
2823         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2824         if (ret < 0)
2825                 goto out;
2826
2827         if (ret > 0) {
2828                 ret = -ENOENT;
2829                 goto out;
2830         }
2831
2832         leaf = path->nodes[0];
2833         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2834
2835         btrfs_set_device_id(leaf, dev_item, device->devid);
2836         btrfs_set_device_type(leaf, dev_item, device->type);
2837         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2838         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2839         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2840         btrfs_set_device_total_bytes(leaf, dev_item,
2841                                      btrfs_device_get_disk_total_bytes(device));
2842         btrfs_set_device_bytes_used(leaf, dev_item,
2843                                     btrfs_device_get_bytes_used(device));
2844         btrfs_mark_buffer_dirty(leaf);
2845
2846 out:
2847         btrfs_free_path(path);
2848         return ret;
2849 }
2850
2851 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2852                       struct btrfs_device *device, u64 new_size)
2853 {
2854         struct btrfs_fs_info *fs_info = device->fs_info;
2855         struct btrfs_super_block *super_copy = fs_info->super_copy;
2856         struct btrfs_fs_devices *fs_devices;
2857         u64 old_total;
2858         u64 diff;
2859
2860         if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2861                 return -EACCES;
2862
2863         new_size = round_down(new_size, fs_info->sectorsize);
2864
2865         mutex_lock(&fs_info->chunk_mutex);
2866         old_total = btrfs_super_total_bytes(super_copy);
2867         diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2868
2869         if (new_size <= device->total_bytes ||
2870             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2871                 mutex_unlock(&fs_info->chunk_mutex);
2872                 return -EINVAL;
2873         }
2874
2875         fs_devices = fs_info->fs_devices;
2876
2877         btrfs_set_super_total_bytes(super_copy,
2878                         round_down(old_total + diff, fs_info->sectorsize));
2879         device->fs_devices->total_rw_bytes += diff;
2880
2881         btrfs_device_set_total_bytes(device, new_size);
2882         btrfs_device_set_disk_total_bytes(device, new_size);
2883         btrfs_clear_space_info_full(device->fs_info);
2884         if (list_empty(&device->resized_list))
2885                 list_add_tail(&device->resized_list,
2886                               &fs_devices->resized_devices);
2887         mutex_unlock(&fs_info->chunk_mutex);
2888
2889         return btrfs_update_device(trans, device);
2890 }
2891
2892 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2893 {
2894         struct btrfs_fs_info *fs_info = trans->fs_info;
2895         struct btrfs_root *root = fs_info->chunk_root;
2896         int ret;
2897         struct btrfs_path *path;
2898         struct btrfs_key key;
2899
2900         path = btrfs_alloc_path();
2901         if (!path)
2902                 return -ENOMEM;
2903
2904         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2905         key.offset = chunk_offset;
2906         key.type = BTRFS_CHUNK_ITEM_KEY;
2907
2908         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2909         if (ret < 0)
2910                 goto out;
2911         else if (ret > 0) { /* Logic error or corruption */
2912                 btrfs_handle_fs_error(fs_info, -ENOENT,
2913                                       "Failed lookup while freeing chunk.");
2914                 ret = -ENOENT;
2915                 goto out;
2916         }
2917
2918         ret = btrfs_del_item(trans, root, path);
2919         if (ret < 0)
2920                 btrfs_handle_fs_error(fs_info, ret,
2921                                       "Failed to delete chunk item.");
2922 out:
2923         btrfs_free_path(path);
2924         return ret;
2925 }
2926
2927 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2928 {
2929         struct btrfs_super_block *super_copy = fs_info->super_copy;
2930         struct btrfs_disk_key *disk_key;
2931         struct btrfs_chunk *chunk;
2932         u8 *ptr;
2933         int ret = 0;
2934         u32 num_stripes;
2935         u32 array_size;
2936         u32 len = 0;
2937         u32 cur;
2938         struct btrfs_key key;
2939
2940         mutex_lock(&fs_info->chunk_mutex);
2941         array_size = btrfs_super_sys_array_size(super_copy);
2942
2943         ptr = super_copy->sys_chunk_array;
2944         cur = 0;
2945
2946         while (cur < array_size) {
2947                 disk_key = (struct btrfs_disk_key *)ptr;
2948                 btrfs_disk_key_to_cpu(&key, disk_key);
2949
2950                 len = sizeof(*disk_key);
2951
2952                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2953                         chunk = (struct btrfs_chunk *)(ptr + len);
2954                         num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2955                         len += btrfs_chunk_item_size(num_stripes);
2956                 } else {
2957                         ret = -EIO;
2958                         break;
2959                 }
2960                 if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2961                     key.offset == chunk_offset) {
2962                         memmove(ptr, ptr + len, array_size - (cur + len));
2963                         array_size -= len;
2964                         btrfs_set_super_sys_array_size(super_copy, array_size);
2965                 } else {
2966                         ptr += len;
2967                         cur += len;
2968                 }
2969         }
2970         mutex_unlock(&fs_info->chunk_mutex);
2971         return ret;
2972 }
2973
2974 /*
2975  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2976  * @logical: Logical block offset in bytes.
2977  * @length: Length of extent in bytes.
2978  *
2979  * Return: Chunk mapping or ERR_PTR.
2980  */
2981 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
2982                                        u64 logical, u64 length)
2983 {
2984         struct extent_map_tree *em_tree;
2985         struct extent_map *em;
2986
2987         em_tree = &fs_info->mapping_tree.map_tree;
2988         read_lock(&em_tree->lock);
2989         em = lookup_extent_mapping(em_tree, logical, length);
2990         read_unlock(&em_tree->lock);
2991
2992         if (!em) {
2993                 btrfs_crit(fs_info, "unable to find logical %llu length %llu",
2994                            logical, length);
2995                 return ERR_PTR(-EINVAL);
2996         }
2997
2998         if (em->start > logical || em->start + em->len < logical) {
2999                 btrfs_crit(fs_info,
3000                            "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
3001                            logical, length, em->start, em->start + em->len);
3002                 free_extent_map(em);
3003                 return ERR_PTR(-EINVAL);
3004         }
3005
3006         /* callers are responsible for dropping em's ref. */
3007         return em;
3008 }
3009
3010 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3011 {
3012         struct btrfs_fs_info *fs_info = trans->fs_info;
3013         struct extent_map *em;
3014         struct map_lookup *map;
3015         u64 dev_extent_len = 0;
3016         int i, ret = 0;
3017         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
3018
3019         em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
3020         if (IS_ERR(em)) {
3021                 /*
3022                  * This is a logic error, but we don't want to just rely on the
3023                  * user having built with ASSERT enabled, so if ASSERT doesn't
3024                  * do anything we still error out.
3025                  */
3026                 ASSERT(0);
3027                 return PTR_ERR(em);
3028         }
3029         map = em->map_lookup;
3030         mutex_lock(&fs_info->chunk_mutex);
3031         check_system_chunk(trans, map->type);
3032         mutex_unlock(&fs_info->chunk_mutex);
3033
3034         /*
3035          * Take the device list mutex to prevent races with the final phase of
3036          * a device replace operation that replaces the device object associated
3037          * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
3038          */
3039         mutex_lock(&fs_devices->device_list_mutex);
3040         for (i = 0; i < map->num_stripes; i++) {
3041                 struct btrfs_device *device = map->stripes[i].dev;
3042                 ret = btrfs_free_dev_extent(trans, device,
3043                                             map->stripes[i].physical,
3044                                             &dev_extent_len);
3045                 if (ret) {
3046                         mutex_unlock(&fs_devices->device_list_mutex);
3047                         btrfs_abort_transaction(trans, ret);
3048                         goto out;
3049                 }
3050
3051                 if (device->bytes_used > 0) {
3052                         mutex_lock(&fs_info->chunk_mutex);
3053                         btrfs_device_set_bytes_used(device,
3054                                         device->bytes_used - dev_extent_len);
3055                         atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
3056                         btrfs_clear_space_info_full(fs_info);
3057                         mutex_unlock(&fs_info->chunk_mutex);
3058                 }
3059
3060                 ret = btrfs_update_device(trans, device);
3061                 if (ret) {
3062                         mutex_unlock(&fs_devices->device_list_mutex);
3063                         btrfs_abort_transaction(trans, ret);
3064                         goto out;
3065                 }
3066         }
3067         mutex_unlock(&fs_devices->device_list_mutex);
3068
3069         ret = btrfs_free_chunk(trans, chunk_offset);
3070         if (ret) {
3071                 btrfs_abort_transaction(trans, ret);
3072                 goto out;
3073         }
3074
3075         trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
3076
3077         if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3078                 ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3079                 if (ret) {
3080                         btrfs_abort_transaction(trans, ret);
3081                         goto out;
3082                 }
3083         }
3084
3085         ret = btrfs_remove_block_group(trans, chunk_offset, em);
3086         if (ret) {
3087                 btrfs_abort_transaction(trans, ret);
3088                 goto out;
3089         }
3090
3091 out:
3092         /* once for us */
3093         free_extent_map(em);
3094         return ret;
3095 }
3096
3097 static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3098 {
3099         struct btrfs_root *root = fs_info->chunk_root;
3100         struct btrfs_trans_handle *trans;
3101         int ret;
3102
3103         /*
3104          * Prevent races with automatic removal of unused block groups.
3105          * After we relocate and before we remove the chunk with offset
3106          * chunk_offset, automatic removal of the block group can kick in,
3107          * resulting in a failure when calling btrfs_remove_chunk() below.
3108          *
3109          * Make sure to acquire this mutex before doing a tree search (dev
3110          * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3111          * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3112          * we release the path used to search the chunk/dev tree and before
3113          * the current task acquires this mutex and calls us.
3114          */
3115         lockdep_assert_held(&fs_info->delete_unused_bgs_mutex);
3116
3117         ret = btrfs_can_relocate(fs_info, chunk_offset);
3118         if (ret)
3119                 return -ENOSPC;
3120
3121         /* step one, relocate all the extents inside this chunk */
3122         btrfs_scrub_pause(fs_info);
3123         ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3124         btrfs_scrub_continue(fs_info);
3125         if (ret)
3126                 return ret;
3127
3128         /*
3129          * We add the kobjects here (and after forcing data chunk creation)
3130          * since relocation is the only place we'll create chunks of a new
3131          * type at runtime.  The only place where we'll remove the last
3132          * chunk of a type is the call immediately below this one.  Even
3133          * so, we're protected against races with the cleaner thread since
3134          * we're covered by the delete_unused_bgs_mutex.
3135          */
3136         btrfs_add_raid_kobjects(fs_info);
3137
3138         trans = btrfs_start_trans_remove_block_group(root->fs_info,
3139                                                      chunk_offset);
3140         if (IS_ERR(trans)) {
3141                 ret = PTR_ERR(trans);
3142                 btrfs_handle_fs_error(root->fs_info, ret, NULL);
3143                 return ret;
3144         }
3145
3146         /*
3147          * step two, delete the device extents and the
3148          * chunk tree entries
3149          */
3150         ret = btrfs_remove_chunk(trans, chunk_offset);
3151         btrfs_end_transaction(trans);
3152         return ret;
3153 }
3154
3155 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3156 {
3157         struct btrfs_root *chunk_root = fs_info->chunk_root;
3158         struct btrfs_path *path;
3159         struct extent_buffer *leaf;
3160         struct btrfs_chunk *chunk;
3161         struct btrfs_key key;
3162         struct btrfs_key found_key;
3163         u64 chunk_type;
3164         bool retried = false;
3165         int failed = 0;
3166         int ret;
3167
3168         path = btrfs_alloc_path();
3169         if (!path)
3170                 return -ENOMEM;
3171
3172 again:
3173         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3174         key.offset = (u64)-1;
3175         key.type = BTRFS_CHUNK_ITEM_KEY;
3176
3177         while (1) {
3178                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
3179                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3180                 if (ret < 0) {
3181                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3182                         goto error;
3183                 }
3184                 BUG_ON(ret == 0); /* Corruption */
3185
3186                 ret = btrfs_previous_item(chunk_root, path, key.objectid,
3187                                           key.type);
3188                 if (ret)
3189                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3190                 if (ret < 0)
3191                         goto error;
3192                 if (ret > 0)
3193                         break;
3194
3195                 leaf = path->nodes[0];
3196                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3197
3198                 chunk = btrfs_item_ptr(leaf, path->slots[0],
3199                                        struct btrfs_chunk);
3200                 chunk_type = btrfs_chunk_type(leaf, chunk);
3201                 btrfs_release_path(path);
3202
3203                 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3204                         ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3205                         if (ret == -ENOSPC)
3206                                 failed++;
3207                         else
3208