btrfs: raid56: allow the exact minimum number of devices for balance convert
[sfrench/cifs-2.6.git] / fs / btrfs / volumes.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/ratelimit.h>
12 #include <linux/kthread.h>
13 #include <linux/raid/pq.h>
14 #include <linux/semaphore.h>
15 #include <linux/uuid.h>
16 #include <linux/list_sort.h>
17 #include "ctree.h"
18 #include "extent_map.h"
19 #include "disk-io.h"
20 #include "transaction.h"
21 #include "print-tree.h"
22 #include "volumes.h"
23 #include "raid56.h"
24 #include "async-thread.h"
25 #include "check-integrity.h"
26 #include "rcu-string.h"
27 #include "math.h"
28 #include "dev-replace.h"
29 #include "sysfs.h"
30 #include "tree-checker.h"
31
32 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
33         [BTRFS_RAID_RAID10] = {
34                 .sub_stripes    = 2,
35                 .dev_stripes    = 1,
36                 .devs_max       = 0,    /* 0 == as many as possible */
37                 .devs_min       = 4,
38                 .tolerated_failures = 1,
39                 .devs_increment = 2,
40                 .ncopies        = 2,
41                 .nparity        = 0,
42                 .raid_name      = "raid10",
43                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID10,
44                 .mindev_error   = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
45         },
46         [BTRFS_RAID_RAID1] = {
47                 .sub_stripes    = 1,
48                 .dev_stripes    = 1,
49                 .devs_max       = 2,
50                 .devs_min       = 2,
51                 .tolerated_failures = 1,
52                 .devs_increment = 2,
53                 .ncopies        = 2,
54                 .nparity        = 0,
55                 .raid_name      = "raid1",
56                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1,
57                 .mindev_error   = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
58         },
59         [BTRFS_RAID_DUP] = {
60                 .sub_stripes    = 1,
61                 .dev_stripes    = 2,
62                 .devs_max       = 1,
63                 .devs_min       = 1,
64                 .tolerated_failures = 0,
65                 .devs_increment = 1,
66                 .ncopies        = 2,
67                 .nparity        = 0,
68                 .raid_name      = "dup",
69                 .bg_flag        = BTRFS_BLOCK_GROUP_DUP,
70                 .mindev_error   = 0,
71         },
72         [BTRFS_RAID_RAID0] = {
73                 .sub_stripes    = 1,
74                 .dev_stripes    = 1,
75                 .devs_max       = 0,
76                 .devs_min       = 2,
77                 .tolerated_failures = 0,
78                 .devs_increment = 1,
79                 .ncopies        = 1,
80                 .nparity        = 0,
81                 .raid_name      = "raid0",
82                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID0,
83                 .mindev_error   = 0,
84         },
85         [BTRFS_RAID_SINGLE] = {
86                 .sub_stripes    = 1,
87                 .dev_stripes    = 1,
88                 .devs_max       = 1,
89                 .devs_min       = 1,
90                 .tolerated_failures = 0,
91                 .devs_increment = 1,
92                 .ncopies        = 1,
93                 .nparity        = 0,
94                 .raid_name      = "single",
95                 .bg_flag        = 0,
96                 .mindev_error   = 0,
97         },
98         [BTRFS_RAID_RAID5] = {
99                 .sub_stripes    = 1,
100                 .dev_stripes    = 1,
101                 .devs_max       = 0,
102                 .devs_min       = 2,
103                 .tolerated_failures = 1,
104                 .devs_increment = 1,
105                 .ncopies        = 1,
106                 .nparity        = 1,
107                 .raid_name      = "raid5",
108                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID5,
109                 .mindev_error   = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
110         },
111         [BTRFS_RAID_RAID6] = {
112                 .sub_stripes    = 1,
113                 .dev_stripes    = 1,
114                 .devs_max       = 0,
115                 .devs_min       = 3,
116                 .tolerated_failures = 2,
117                 .devs_increment = 1,
118                 .ncopies        = 1,
119                 .nparity        = 2,
120                 .raid_name      = "raid6",
121                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID6,
122                 .mindev_error   = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
123         },
124 };
125
126 const char *get_raid_name(enum btrfs_raid_types type)
127 {
128         if (type >= BTRFS_NR_RAID_TYPES)
129                 return NULL;
130
131         return btrfs_raid_array[type].raid_name;
132 }
133
134 /*
135  * Fill @buf with textual description of @bg_flags, no more than @size_buf
136  * bytes including terminating null byte.
137  */
138 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
139 {
140         int i;
141         int ret;
142         char *bp = buf;
143         u64 flags = bg_flags;
144         u32 size_bp = size_buf;
145
146         if (!flags) {
147                 strcpy(bp, "NONE");
148                 return;
149         }
150
151 #define DESCRIBE_FLAG(flag, desc)                                               \
152         do {                                                            \
153                 if (flags & (flag)) {                                   \
154                         ret = snprintf(bp, size_bp, "%s|", (desc));     \
155                         if (ret < 0 || ret >= size_bp)                  \
156                                 goto out_overflow;                      \
157                         size_bp -= ret;                                 \
158                         bp += ret;                                      \
159                         flags &= ~(flag);                               \
160                 }                                                       \
161         } while (0)
162
163         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
164         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
165         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
166
167         DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
168         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
169                 DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
170                               btrfs_raid_array[i].raid_name);
171 #undef DESCRIBE_FLAG
172
173         if (flags) {
174                 ret = snprintf(bp, size_bp, "0x%llx|", flags);
175                 size_bp -= ret;
176         }
177
178         if (size_bp < size_buf)
179                 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
180
181         /*
182          * The text is trimmed, it's up to the caller to provide sufficiently
183          * large buffer
184          */
185 out_overflow:;
186 }
187
188 static int init_first_rw_device(struct btrfs_trans_handle *trans);
189 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
190 static void __btrfs_reset_dev_stats(struct btrfs_device *dev);
191 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
192 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
193 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
194                              enum btrfs_map_op op,
195                              u64 logical, u64 *length,
196                              struct btrfs_bio **bbio_ret,
197                              int mirror_num, int need_raid_map);
198
199 /*
200  * Device locking
201  * ==============
202  *
203  * There are several mutexes that protect manipulation of devices and low-level
204  * structures like chunks but not block groups, extents or files
205  *
206  * uuid_mutex (global lock)
207  * ------------------------
208  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
209  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
210  * device) or requested by the device= mount option
211  *
212  * the mutex can be very coarse and can cover long-running operations
213  *
214  * protects: updates to fs_devices counters like missing devices, rw devices,
215  * seeding, structure cloning, opening/closing devices at mount/umount time
216  *
217  * global::fs_devs - add, remove, updates to the global list
218  *
219  * does not protect: manipulation of the fs_devices::devices list!
220  *
221  * btrfs_device::name - renames (write side), read is RCU
222  *
223  * fs_devices::device_list_mutex (per-fs, with RCU)
224  * ------------------------------------------------
225  * protects updates to fs_devices::devices, ie. adding and deleting
226  *
227  * simple list traversal with read-only actions can be done with RCU protection
228  *
229  * may be used to exclude some operations from running concurrently without any
230  * modifications to the list (see write_all_supers)
231  *
232  * balance_mutex
233  * -------------
234  * protects balance structures (status, state) and context accessed from
235  * several places (internally, ioctl)
236  *
237  * chunk_mutex
238  * -----------
239  * protects chunks, adding or removing during allocation, trim or when a new
240  * device is added/removed
241  *
242  * cleaner_mutex
243  * -------------
244  * a big lock that is held by the cleaner thread and prevents running subvolume
245  * cleaning together with relocation or delayed iputs
246  *
247  *
248  * Lock nesting
249  * ============
250  *
251  * uuid_mutex
252  *   volume_mutex
253  *     device_list_mutex
254  *       chunk_mutex
255  *     balance_mutex
256  *
257  *
258  * Exclusive operations, BTRFS_FS_EXCL_OP
259  * ======================================
260  *
261  * Maintains the exclusivity of the following operations that apply to the
262  * whole filesystem and cannot run in parallel.
263  *
264  * - Balance (*)
265  * - Device add
266  * - Device remove
267  * - Device replace (*)
268  * - Resize
269  *
270  * The device operations (as above) can be in one of the following states:
271  *
272  * - Running state
273  * - Paused state
274  * - Completed state
275  *
276  * Only device operations marked with (*) can go into the Paused state for the
277  * following reasons:
278  *
279  * - ioctl (only Balance can be Paused through ioctl)
280  * - filesystem remounted as read-only
281  * - filesystem unmounted and mounted as read-only
282  * - system power-cycle and filesystem mounted as read-only
283  * - filesystem or device errors leading to forced read-only
284  *
285  * BTRFS_FS_EXCL_OP flag is set and cleared using atomic operations.
286  * During the course of Paused state, the BTRFS_FS_EXCL_OP remains set.
287  * A device operation in Paused or Running state can be canceled or resumed
288  * either by ioctl (Balance only) or when remounted as read-write.
289  * BTRFS_FS_EXCL_OP flag is cleared when the device operation is canceled or
290  * completed.
291  */
292
293 DEFINE_MUTEX(uuid_mutex);
294 static LIST_HEAD(fs_uuids);
295 struct list_head *btrfs_get_fs_uuids(void)
296 {
297         return &fs_uuids;
298 }
299
300 /*
301  * alloc_fs_devices - allocate struct btrfs_fs_devices
302  * @fsid:               if not NULL, copy the UUID to fs_devices::fsid
303  * @metadata_fsid:      if not NULL, copy the UUID to fs_devices::metadata_fsid
304  *
305  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
306  * The returned struct is not linked onto any lists and can be destroyed with
307  * kfree() right away.
308  */
309 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
310                                                  const u8 *metadata_fsid)
311 {
312         struct btrfs_fs_devices *fs_devs;
313
314         fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
315         if (!fs_devs)
316                 return ERR_PTR(-ENOMEM);
317
318         mutex_init(&fs_devs->device_list_mutex);
319
320         INIT_LIST_HEAD(&fs_devs->devices);
321         INIT_LIST_HEAD(&fs_devs->alloc_list);
322         INIT_LIST_HEAD(&fs_devs->fs_list);
323         if (fsid)
324                 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
325
326         if (metadata_fsid)
327                 memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
328         else if (fsid)
329                 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
330
331         return fs_devs;
332 }
333
334 void btrfs_free_device(struct btrfs_device *device)
335 {
336         WARN_ON(!list_empty(&device->post_commit_list));
337         rcu_string_free(device->name);
338         extent_io_tree_release(&device->alloc_state);
339         bio_put(device->flush_bio);
340         kfree(device);
341 }
342
343 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
344 {
345         struct btrfs_device *device;
346         WARN_ON(fs_devices->opened);
347         while (!list_empty(&fs_devices->devices)) {
348                 device = list_entry(fs_devices->devices.next,
349                                     struct btrfs_device, dev_list);
350                 list_del(&device->dev_list);
351                 btrfs_free_device(device);
352         }
353         kfree(fs_devices);
354 }
355
356 static void btrfs_kobject_uevent(struct block_device *bdev,
357                                  enum kobject_action action)
358 {
359         int ret;
360
361         ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
362         if (ret)
363                 pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
364                         action,
365                         kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
366                         &disk_to_dev(bdev->bd_disk)->kobj);
367 }
368
369 void __exit btrfs_cleanup_fs_uuids(void)
370 {
371         struct btrfs_fs_devices *fs_devices;
372
373         while (!list_empty(&fs_uuids)) {
374                 fs_devices = list_entry(fs_uuids.next,
375                                         struct btrfs_fs_devices, fs_list);
376                 list_del(&fs_devices->fs_list);
377                 free_fs_devices(fs_devices);
378         }
379 }
380
381 /*
382  * Returns a pointer to a new btrfs_device on success; ERR_PTR() on error.
383  * Returned struct is not linked onto any lists and must be destroyed using
384  * btrfs_free_device.
385  */
386 static struct btrfs_device *__alloc_device(void)
387 {
388         struct btrfs_device *dev;
389
390         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
391         if (!dev)
392                 return ERR_PTR(-ENOMEM);
393
394         /*
395          * Preallocate a bio that's always going to be used for flushing device
396          * barriers and matches the device lifespan
397          */
398         dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
399         if (!dev->flush_bio) {
400                 kfree(dev);
401                 return ERR_PTR(-ENOMEM);
402         }
403
404         INIT_LIST_HEAD(&dev->dev_list);
405         INIT_LIST_HEAD(&dev->dev_alloc_list);
406         INIT_LIST_HEAD(&dev->post_commit_list);
407
408         spin_lock_init(&dev->io_lock);
409
410         atomic_set(&dev->reada_in_flight, 0);
411         atomic_set(&dev->dev_stats_ccnt, 0);
412         btrfs_device_data_ordered_init(dev);
413         INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
414         INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
415         extent_io_tree_init(NULL, &dev->alloc_state, 0, NULL);
416
417         return dev;
418 }
419
420 static noinline struct btrfs_fs_devices *find_fsid(
421                 const u8 *fsid, const u8 *metadata_fsid)
422 {
423         struct btrfs_fs_devices *fs_devices;
424
425         ASSERT(fsid);
426
427         if (metadata_fsid) {
428                 /*
429                  * Handle scanned device having completed its fsid change but
430                  * belonging to a fs_devices that was created by first scanning
431                  * a device which didn't have its fsid/metadata_uuid changed
432                  * at all and the CHANGING_FSID_V2 flag set.
433                  */
434                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
435                         if (fs_devices->fsid_change &&
436                             memcmp(metadata_fsid, fs_devices->fsid,
437                                    BTRFS_FSID_SIZE) == 0 &&
438                             memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
439                                    BTRFS_FSID_SIZE) == 0) {
440                                 return fs_devices;
441                         }
442                 }
443                 /*
444                  * Handle scanned device having completed its fsid change but
445                  * belonging to a fs_devices that was created by a device that
446                  * has an outdated pair of fsid/metadata_uuid and
447                  * CHANGING_FSID_V2 flag set.
448                  */
449                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
450                         if (fs_devices->fsid_change &&
451                             memcmp(fs_devices->metadata_uuid,
452                                    fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
453                             memcmp(metadata_fsid, fs_devices->metadata_uuid,
454                                    BTRFS_FSID_SIZE) == 0) {
455                                 return fs_devices;
456                         }
457                 }
458         }
459
460         /* Handle non-split brain cases */
461         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
462                 if (metadata_fsid) {
463                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
464                             && memcmp(metadata_fsid, fs_devices->metadata_uuid,
465                                       BTRFS_FSID_SIZE) == 0)
466                                 return fs_devices;
467                 } else {
468                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
469                                 return fs_devices;
470                 }
471         }
472         return NULL;
473 }
474
475 static int
476 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
477                       int flush, struct block_device **bdev,
478                       struct buffer_head **bh)
479 {
480         int ret;
481
482         *bdev = blkdev_get_by_path(device_path, flags, holder);
483
484         if (IS_ERR(*bdev)) {
485                 ret = PTR_ERR(*bdev);
486                 goto error;
487         }
488
489         if (flush)
490                 filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
491         ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
492         if (ret) {
493                 blkdev_put(*bdev, flags);
494                 goto error;
495         }
496         invalidate_bdev(*bdev);
497         *bh = btrfs_read_dev_super(*bdev);
498         if (IS_ERR(*bh)) {
499                 ret = PTR_ERR(*bh);
500                 blkdev_put(*bdev, flags);
501                 goto error;
502         }
503
504         return 0;
505
506 error:
507         *bdev = NULL;
508         *bh = NULL;
509         return ret;
510 }
511
512 static void requeue_list(struct btrfs_pending_bios *pending_bios,
513                         struct bio *head, struct bio *tail)
514 {
515
516         struct bio *old_head;
517
518         old_head = pending_bios->head;
519         pending_bios->head = head;
520         if (pending_bios->tail)
521                 tail->bi_next = old_head;
522         else
523                 pending_bios->tail = tail;
524 }
525
526 /*
527  * we try to collect pending bios for a device so we don't get a large
528  * number of procs sending bios down to the same device.  This greatly
529  * improves the schedulers ability to collect and merge the bios.
530  *
531  * But, it also turns into a long list of bios to process and that is sure
532  * to eventually make the worker thread block.  The solution here is to
533  * make some progress and then put this work struct back at the end of
534  * the list if the block device is congested.  This way, multiple devices
535  * can make progress from a single worker thread.
536  */
537 static noinline void run_scheduled_bios(struct btrfs_device *device)
538 {
539         struct btrfs_fs_info *fs_info = device->fs_info;
540         struct bio *pending;
541         struct backing_dev_info *bdi;
542         struct btrfs_pending_bios *pending_bios;
543         struct bio *tail;
544         struct bio *cur;
545         int again = 0;
546         unsigned long num_run;
547         unsigned long batch_run = 0;
548         unsigned long last_waited = 0;
549         int force_reg = 0;
550         int sync_pending = 0;
551         struct blk_plug plug;
552
553         /*
554          * this function runs all the bios we've collected for
555          * a particular device.  We don't want to wander off to
556          * another device without first sending all of these down.
557          * So, setup a plug here and finish it off before we return
558          */
559         blk_start_plug(&plug);
560
561         bdi = device->bdev->bd_bdi;
562
563 loop:
564         spin_lock(&device->io_lock);
565
566 loop_lock:
567         num_run = 0;
568
569         /* take all the bios off the list at once and process them
570          * later on (without the lock held).  But, remember the
571          * tail and other pointers so the bios can be properly reinserted
572          * into the list if we hit congestion
573          */
574         if (!force_reg && device->pending_sync_bios.head) {
575                 pending_bios = &device->pending_sync_bios;
576                 force_reg = 1;
577         } else {
578                 pending_bios = &device->pending_bios;
579                 force_reg = 0;
580         }
581
582         pending = pending_bios->head;
583         tail = pending_bios->tail;
584         WARN_ON(pending && !tail);
585
586         /*
587          * if pending was null this time around, no bios need processing
588          * at all and we can stop.  Otherwise it'll loop back up again
589          * and do an additional check so no bios are missed.
590          *
591          * device->running_pending is used to synchronize with the
592          * schedule_bio code.
593          */
594         if (device->pending_sync_bios.head == NULL &&
595             device->pending_bios.head == NULL) {
596                 again = 0;
597                 device->running_pending = 0;
598         } else {
599                 again = 1;
600                 device->running_pending = 1;
601         }
602
603         pending_bios->head = NULL;
604         pending_bios->tail = NULL;
605
606         spin_unlock(&device->io_lock);
607
608         while (pending) {
609
610                 rmb();
611                 /* we want to work on both lists, but do more bios on the
612                  * sync list than the regular list
613                  */
614                 if ((num_run > 32 &&
615                     pending_bios != &device->pending_sync_bios &&
616                     device->pending_sync_bios.head) ||
617                    (num_run > 64 && pending_bios == &device->pending_sync_bios &&
618                     device->pending_bios.head)) {
619                         spin_lock(&device->io_lock);
620                         requeue_list(pending_bios, pending, tail);
621                         goto loop_lock;
622                 }
623
624                 cur = pending;
625                 pending = pending->bi_next;
626                 cur->bi_next = NULL;
627
628                 BUG_ON(atomic_read(&cur->__bi_cnt) == 0);
629
630                 /*
631                  * if we're doing the sync list, record that our
632                  * plug has some sync requests on it
633                  *
634                  * If we're doing the regular list and there are
635                  * sync requests sitting around, unplug before
636                  * we add more
637                  */
638                 if (pending_bios == &device->pending_sync_bios) {
639                         sync_pending = 1;
640                 } else if (sync_pending) {
641                         blk_finish_plug(&plug);
642                         blk_start_plug(&plug);
643                         sync_pending = 0;
644                 }
645
646                 btrfsic_submit_bio(cur);
647                 num_run++;
648                 batch_run++;
649
650                 cond_resched();
651
652                 /*
653                  * we made progress, there is more work to do and the bdi
654                  * is now congested.  Back off and let other work structs
655                  * run instead
656                  */
657                 if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
658                     fs_info->fs_devices->open_devices > 1) {
659                         struct io_context *ioc;
660
661                         ioc = current->io_context;
662
663                         /*
664                          * the main goal here is that we don't want to
665                          * block if we're going to be able to submit
666                          * more requests without blocking.
667                          *
668                          * This code does two great things, it pokes into
669                          * the elevator code from a filesystem _and_
670                          * it makes assumptions about how batching works.
671                          */
672                         if (ioc && ioc->nr_batch_requests > 0 &&
673                             time_before(jiffies, ioc->last_waited + HZ/50UL) &&
674                             (last_waited == 0 ||
675                              ioc->last_waited == last_waited)) {
676                                 /*
677                                  * we want to go through our batch of
678                                  * requests and stop.  So, we copy out
679                                  * the ioc->last_waited time and test
680                                  * against it before looping
681                                  */
682                                 last_waited = ioc->last_waited;
683                                 cond_resched();
684                                 continue;
685                         }
686                         spin_lock(&device->io_lock);
687                         requeue_list(pending_bios, pending, tail);
688                         device->running_pending = 1;
689
690                         spin_unlock(&device->io_lock);
691                         btrfs_queue_work(fs_info->submit_workers,
692                                          &device->work);
693                         goto done;
694                 }
695         }
696
697         cond_resched();
698         if (again)
699                 goto loop;
700
701         spin_lock(&device->io_lock);
702         if (device->pending_bios.head || device->pending_sync_bios.head)
703                 goto loop_lock;
704         spin_unlock(&device->io_lock);
705
706 done:
707         blk_finish_plug(&plug);
708 }
709
710 static void pending_bios_fn(struct btrfs_work *work)
711 {
712         struct btrfs_device *device;
713
714         device = container_of(work, struct btrfs_device, work);
715         run_scheduled_bios(device);
716 }
717
718 static bool device_path_matched(const char *path, struct btrfs_device *device)
719 {
720         int found;
721
722         rcu_read_lock();
723         found = strcmp(rcu_str_deref(device->name), path);
724         rcu_read_unlock();
725
726         return found == 0;
727 }
728
729 /*
730  *  Search and remove all stale (devices which are not mounted) devices.
731  *  When both inputs are NULL, it will search and release all stale devices.
732  *  path:       Optional. When provided will it release all unmounted devices
733  *              matching this path only.
734  *  skip_dev:   Optional. Will skip this device when searching for the stale
735  *              devices.
736  *  Return:     0 for success or if @path is NULL.
737  *              -EBUSY if @path is a mounted device.
738  *              -ENOENT if @path does not match any device in the list.
739  */
740 static int btrfs_free_stale_devices(const char *path,
741                                      struct btrfs_device *skip_device)
742 {
743         struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
744         struct btrfs_device *device, *tmp_device;
745         int ret = 0;
746
747         if (path)
748                 ret = -ENOENT;
749
750         list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
751
752                 mutex_lock(&fs_devices->device_list_mutex);
753                 list_for_each_entry_safe(device, tmp_device,
754                                          &fs_devices->devices, dev_list) {
755                         if (skip_device && skip_device == device)
756                                 continue;
757                         if (path && !device->name)
758                                 continue;
759                         if (path && !device_path_matched(path, device))
760                                 continue;
761                         if (fs_devices->opened) {
762                                 /* for an already deleted device return 0 */
763                                 if (path && ret != 0)
764                                         ret = -EBUSY;
765                                 break;
766                         }
767
768                         /* delete the stale device */
769                         fs_devices->num_devices--;
770                         list_del(&device->dev_list);
771                         btrfs_free_device(device);
772
773                         ret = 0;
774                         if (fs_devices->num_devices == 0)
775                                 break;
776                 }
777                 mutex_unlock(&fs_devices->device_list_mutex);
778
779                 if (fs_devices->num_devices == 0) {
780                         btrfs_sysfs_remove_fsid(fs_devices);
781                         list_del(&fs_devices->fs_list);
782                         free_fs_devices(fs_devices);
783                 }
784         }
785
786         return ret;
787 }
788
789 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
790                         struct btrfs_device *device, fmode_t flags,
791                         void *holder)
792 {
793         struct request_queue *q;
794         struct block_device *bdev;
795         struct buffer_head *bh;
796         struct btrfs_super_block *disk_super;
797         u64 devid;
798         int ret;
799
800         if (device->bdev)
801                 return -EINVAL;
802         if (!device->name)
803                 return -EINVAL;
804
805         ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
806                                     &bdev, &bh);
807         if (ret)
808                 return ret;
809
810         disk_super = (struct btrfs_super_block *)bh->b_data;
811         devid = btrfs_stack_device_id(&disk_super->dev_item);
812         if (devid != device->devid)
813                 goto error_brelse;
814
815         if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
816                 goto error_brelse;
817
818         device->generation = btrfs_super_generation(disk_super);
819
820         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
821                 if (btrfs_super_incompat_flags(disk_super) &
822                     BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
823                         pr_err(
824                 "BTRFS: Invalid seeding and uuid-changed device detected\n");
825                         goto error_brelse;
826                 }
827
828                 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
829                 fs_devices->seeding = 1;
830         } else {
831                 if (bdev_read_only(bdev))
832                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
833                 else
834                         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
835         }
836
837         q = bdev_get_queue(bdev);
838         if (!blk_queue_nonrot(q))
839                 fs_devices->rotating = 1;
840
841         device->bdev = bdev;
842         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
843         device->mode = flags;
844
845         fs_devices->open_devices++;
846         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
847             device->devid != BTRFS_DEV_REPLACE_DEVID) {
848                 fs_devices->rw_devices++;
849                 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
850         }
851         brelse(bh);
852
853         return 0;
854
855 error_brelse:
856         brelse(bh);
857         blkdev_put(bdev, flags);
858
859         return -EINVAL;
860 }
861
862 /*
863  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
864  * being created with a disk that has already completed its fsid change.
865  */
866 static struct btrfs_fs_devices *find_fsid_inprogress(
867                                         struct btrfs_super_block *disk_super)
868 {
869         struct btrfs_fs_devices *fs_devices;
870
871         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
872                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
873                            BTRFS_FSID_SIZE) != 0 &&
874                     memcmp(fs_devices->metadata_uuid, disk_super->fsid,
875                            BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
876                         return fs_devices;
877                 }
878         }
879
880         return NULL;
881 }
882
883
884 static struct btrfs_fs_devices *find_fsid_changed(
885                                         struct btrfs_super_block *disk_super)
886 {
887         struct btrfs_fs_devices *fs_devices;
888
889         /*
890          * Handles the case where scanned device is part of an fs that had
891          * multiple successful changes of FSID but curently device didn't
892          * observe it. Meaning our fsid will be different than theirs.
893          */
894         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
895                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
896                            BTRFS_FSID_SIZE) != 0 &&
897                     memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
898                            BTRFS_FSID_SIZE) == 0 &&
899                     memcmp(fs_devices->fsid, disk_super->fsid,
900                            BTRFS_FSID_SIZE) != 0) {
901                         return fs_devices;
902                 }
903         }
904
905         return NULL;
906 }
907 /*
908  * Add new device to list of registered devices
909  *
910  * Returns:
911  * device pointer which was just added or updated when successful
912  * error pointer when failed
913  */
914 static noinline struct btrfs_device *device_list_add(const char *path,
915                            struct btrfs_super_block *disk_super,
916                            bool *new_device_added)
917 {
918         struct btrfs_device *device;
919         struct btrfs_fs_devices *fs_devices = NULL;
920         struct rcu_string *name;
921         u64 found_transid = btrfs_super_generation(disk_super);
922         u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
923         bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
924                 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
925         bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
926                                         BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
927
928         if (fsid_change_in_progress) {
929                 if (!has_metadata_uuid) {
930                         /*
931                          * When we have an image which has CHANGING_FSID_V2 set
932                          * it might belong to either a filesystem which has
933                          * disks with completed fsid change or it might belong
934                          * to fs with no UUID changes in effect, handle both.
935                          */
936                         fs_devices = find_fsid_inprogress(disk_super);
937                         if (!fs_devices)
938                                 fs_devices = find_fsid(disk_super->fsid, NULL);
939                 } else {
940                         fs_devices = find_fsid_changed(disk_super);
941                 }
942         } else if (has_metadata_uuid) {
943                 fs_devices = find_fsid(disk_super->fsid,
944                                        disk_super->metadata_uuid);
945         } else {
946                 fs_devices = find_fsid(disk_super->fsid, NULL);
947         }
948
949
950         if (!fs_devices) {
951                 if (has_metadata_uuid)
952                         fs_devices = alloc_fs_devices(disk_super->fsid,
953                                                       disk_super->metadata_uuid);
954                 else
955                         fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
956
957                 if (IS_ERR(fs_devices))
958                         return ERR_CAST(fs_devices);
959
960                 fs_devices->fsid_change = fsid_change_in_progress;
961
962                 mutex_lock(&fs_devices->device_list_mutex);
963                 list_add(&fs_devices->fs_list, &fs_uuids);
964
965                 device = NULL;
966         } else {
967                 mutex_lock(&fs_devices->device_list_mutex);
968                 device = btrfs_find_device(fs_devices, devid,
969                                 disk_super->dev_item.uuid, NULL, false);
970
971                 /*
972                  * If this disk has been pulled into an fs devices created by
973                  * a device which had the CHANGING_FSID_V2 flag then replace the
974                  * metadata_uuid/fsid values of the fs_devices.
975                  */
976                 if (has_metadata_uuid && fs_devices->fsid_change &&
977                     found_transid > fs_devices->latest_generation) {
978                         memcpy(fs_devices->fsid, disk_super->fsid,
979                                         BTRFS_FSID_SIZE);
980                         memcpy(fs_devices->metadata_uuid,
981                                         disk_super->metadata_uuid, BTRFS_FSID_SIZE);
982
983                         fs_devices->fsid_change = false;
984                 }
985         }
986
987         if (!device) {
988                 if (fs_devices->opened) {
989                         mutex_unlock(&fs_devices->device_list_mutex);
990                         return ERR_PTR(-EBUSY);
991                 }
992
993                 device = btrfs_alloc_device(NULL, &devid,
994                                             disk_super->dev_item.uuid);
995                 if (IS_ERR(device)) {
996                         mutex_unlock(&fs_devices->device_list_mutex);
997                         /* we can safely leave the fs_devices entry around */
998                         return device;
999                 }
1000
1001                 name = rcu_string_strdup(path, GFP_NOFS);
1002                 if (!name) {
1003                         btrfs_free_device(device);
1004                         mutex_unlock(&fs_devices->device_list_mutex);
1005                         return ERR_PTR(-ENOMEM);
1006                 }
1007                 rcu_assign_pointer(device->name, name);
1008
1009                 list_add_rcu(&device->dev_list, &fs_devices->devices);
1010                 fs_devices->num_devices++;
1011
1012                 device->fs_devices = fs_devices;
1013                 *new_device_added = true;
1014
1015                 if (disk_super->label[0])
1016                         pr_info("BTRFS: device label %s devid %llu transid %llu %s\n",
1017                                 disk_super->label, devid, found_transid, path);
1018                 else
1019                         pr_info("BTRFS: device fsid %pU devid %llu transid %llu %s\n",
1020                                 disk_super->fsid, devid, found_transid, path);
1021
1022         } else if (!device->name || strcmp(device->name->str, path)) {
1023                 /*
1024                  * When FS is already mounted.
1025                  * 1. If you are here and if the device->name is NULL that
1026                  *    means this device was missing at time of FS mount.
1027                  * 2. If you are here and if the device->name is different
1028                  *    from 'path' that means either
1029                  *      a. The same device disappeared and reappeared with
1030                  *         different name. or
1031                  *      b. The missing-disk-which-was-replaced, has
1032                  *         reappeared now.
1033                  *
1034                  * We must allow 1 and 2a above. But 2b would be a spurious
1035                  * and unintentional.
1036                  *
1037                  * Further in case of 1 and 2a above, the disk at 'path'
1038                  * would have missed some transaction when it was away and
1039                  * in case of 2a the stale bdev has to be updated as well.
1040                  * 2b must not be allowed at all time.
1041                  */
1042
1043                 /*
1044                  * For now, we do allow update to btrfs_fs_device through the
1045                  * btrfs dev scan cli after FS has been mounted.  We're still
1046                  * tracking a problem where systems fail mount by subvolume id
1047                  * when we reject replacement on a mounted FS.
1048                  */
1049                 if (!fs_devices->opened && found_transid < device->generation) {
1050                         /*
1051                          * That is if the FS is _not_ mounted and if you
1052                          * are here, that means there is more than one
1053                          * disk with same uuid and devid.We keep the one
1054                          * with larger generation number or the last-in if
1055                          * generation are equal.
1056                          */
1057                         mutex_unlock(&fs_devices->device_list_mutex);
1058                         return ERR_PTR(-EEXIST);
1059                 }
1060
1061                 /*
1062                  * We are going to replace the device path for a given devid,
1063                  * make sure it's the same device if the device is mounted
1064                  */
1065                 if (device->bdev) {
1066                         struct block_device *path_bdev;
1067
1068                         path_bdev = lookup_bdev(path);
1069                         if (IS_ERR(path_bdev)) {
1070                                 mutex_unlock(&fs_devices->device_list_mutex);
1071                                 return ERR_CAST(path_bdev);
1072                         }
1073
1074                         if (device->bdev != path_bdev) {
1075                                 bdput(path_bdev);
1076                                 mutex_unlock(&fs_devices->device_list_mutex);
1077                                 btrfs_warn_in_rcu(device->fs_info,
1078                         "duplicate device fsid:devid for %pU:%llu old:%s new:%s",
1079                                         disk_super->fsid, devid,
1080                                         rcu_str_deref(device->name), path);
1081                                 return ERR_PTR(-EEXIST);
1082                         }
1083                         bdput(path_bdev);
1084                         btrfs_info_in_rcu(device->fs_info,
1085                                 "device fsid %pU devid %llu moved old:%s new:%s",
1086                                 disk_super->fsid, devid,
1087                                 rcu_str_deref(device->name), path);
1088                 }
1089
1090                 name = rcu_string_strdup(path, GFP_NOFS);
1091                 if (!name) {
1092                         mutex_unlock(&fs_devices->device_list_mutex);
1093                         return ERR_PTR(-ENOMEM);
1094                 }
1095                 rcu_string_free(device->name);
1096                 rcu_assign_pointer(device->name, name);
1097                 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1098                         fs_devices->missing_devices--;
1099                         clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1100                 }
1101         }
1102
1103         /*
1104          * Unmount does not free the btrfs_device struct but would zero
1105          * generation along with most of the other members. So just update
1106          * it back. We need it to pick the disk with largest generation
1107          * (as above).
1108          */
1109         if (!fs_devices->opened) {
1110                 device->generation = found_transid;
1111                 fs_devices->latest_generation = max_t(u64, found_transid,
1112                                                 fs_devices->latest_generation);
1113         }
1114
1115         fs_devices->total_devices = btrfs_super_num_devices(disk_super);
1116
1117         mutex_unlock(&fs_devices->device_list_mutex);
1118         return device;
1119 }
1120
1121 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
1122 {
1123         struct btrfs_fs_devices *fs_devices;
1124         struct btrfs_device *device;
1125         struct btrfs_device *orig_dev;
1126
1127         fs_devices = alloc_fs_devices(orig->fsid, NULL);
1128         if (IS_ERR(fs_devices))
1129                 return fs_devices;
1130
1131         mutex_lock(&orig->device_list_mutex);
1132         fs_devices->total_devices = orig->total_devices;
1133
1134         list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1135                 struct rcu_string *name;
1136
1137                 device = btrfs_alloc_device(NULL, &orig_dev->devid,
1138                                             orig_dev->uuid);
1139                 if (IS_ERR(device))
1140                         goto error;
1141
1142                 /*
1143                  * This is ok to do without rcu read locked because we hold the
1144                  * uuid mutex so nothing we touch in here is going to disappear.
1145                  */
1146                 if (orig_dev->name) {
1147                         name = rcu_string_strdup(orig_dev->name->str,
1148                                         GFP_KERNEL);
1149                         if (!name) {
1150                                 btrfs_free_device(device);
1151                                 goto error;
1152                         }
1153                         rcu_assign_pointer(device->name, name);
1154                 }
1155
1156                 list_add(&device->dev_list, &fs_devices->devices);
1157                 device->fs_devices = fs_devices;
1158                 fs_devices->num_devices++;
1159         }
1160         mutex_unlock(&orig->device_list_mutex);
1161         return fs_devices;
1162 error:
1163         mutex_unlock(&orig->device_list_mutex);
1164         free_fs_devices(fs_devices);
1165         return ERR_PTR(-ENOMEM);
1166 }
1167
1168 /*
1169  * After we have read the system tree and know devids belonging to
1170  * this filesystem, remove the device which does not belong there.
1171  */
1172 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
1173 {
1174         struct btrfs_device *device, *next;
1175         struct btrfs_device *latest_dev = NULL;
1176
1177         mutex_lock(&uuid_mutex);
1178 again:
1179         /* This is the initialized path, it is safe to release the devices. */
1180         list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1181                 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
1182                                                         &device->dev_state)) {
1183                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1184                              &device->dev_state) &&
1185                              (!latest_dev ||
1186                               device->generation > latest_dev->generation)) {
1187                                 latest_dev = device;
1188                         }
1189                         continue;
1190                 }
1191
1192                 if (device->devid == BTRFS_DEV_REPLACE_DEVID) {
1193                         /*
1194                          * In the first step, keep the device which has
1195                          * the correct fsid and the devid that is used
1196                          * for the dev_replace procedure.
1197                          * In the second step, the dev_replace state is
1198                          * read from the device tree and it is known
1199                          * whether the procedure is really active or
1200                          * not, which means whether this device is
1201                          * used or whether it should be removed.
1202                          */
1203                         if (step == 0 || test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1204                                                   &device->dev_state)) {
1205                                 continue;
1206                         }
1207                 }
1208                 if (device->bdev) {
1209                         blkdev_put(device->bdev, device->mode);
1210                         device->bdev = NULL;
1211                         fs_devices->open_devices--;
1212                 }
1213                 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1214                         list_del_init(&device->dev_alloc_list);
1215                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1216                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1217                                       &device->dev_state))
1218                                 fs_devices->rw_devices--;
1219                 }
1220                 list_del_init(&device->dev_list);
1221                 fs_devices->num_devices--;
1222                 btrfs_free_device(device);
1223         }
1224
1225         if (fs_devices->seed) {
1226                 fs_devices = fs_devices->seed;
1227                 goto again;
1228         }
1229
1230         fs_devices->latest_bdev = latest_dev->bdev;
1231
1232         mutex_unlock(&uuid_mutex);
1233 }
1234
1235 static void btrfs_close_bdev(struct btrfs_device *device)
1236 {
1237         if (!device->bdev)
1238                 return;
1239
1240         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1241                 sync_blockdev(device->bdev);
1242                 invalidate_bdev(device->bdev);
1243         }
1244
1245         blkdev_put(device->bdev, device->mode);
1246 }
1247
1248 static void btrfs_close_one_device(struct btrfs_device *device)
1249 {
1250         struct btrfs_fs_devices *fs_devices = device->fs_devices;
1251         struct btrfs_device *new_device;
1252         struct rcu_string *name;
1253
1254         if (device->bdev)
1255                 fs_devices->open_devices--;
1256
1257         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1258             device->devid != BTRFS_DEV_REPLACE_DEVID) {
1259                 list_del_init(&device->dev_alloc_list);
1260                 fs_devices->rw_devices--;
1261         }
1262
1263         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
1264                 fs_devices->missing_devices--;
1265
1266         btrfs_close_bdev(device);
1267
1268         new_device = btrfs_alloc_device(NULL, &device->devid,
1269                                         device->uuid);
1270         BUG_ON(IS_ERR(new_device)); /* -ENOMEM */
1271
1272         /* Safe because we are under uuid_mutex */
1273         if (device->name) {
1274                 name = rcu_string_strdup(device->name->str, GFP_NOFS);
1275                 BUG_ON(!name); /* -ENOMEM */
1276                 rcu_assign_pointer(new_device->name, name);
1277         }
1278
1279         list_replace_rcu(&device->dev_list, &new_device->dev_list);
1280         new_device->fs_devices = device->fs_devices;
1281
1282         synchronize_rcu();
1283         btrfs_free_device(device);
1284 }
1285
1286 static int close_fs_devices(struct btrfs_fs_devices *fs_devices)
1287 {
1288         struct btrfs_device *device, *tmp;
1289
1290         if (--fs_devices->opened > 0)
1291                 return 0;
1292
1293         mutex_lock(&fs_devices->device_list_mutex);
1294         list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) {
1295                 btrfs_close_one_device(device);
1296         }
1297         mutex_unlock(&fs_devices->device_list_mutex);
1298
1299         WARN_ON(fs_devices->open_devices);
1300         WARN_ON(fs_devices->rw_devices);
1301         fs_devices->opened = 0;
1302         fs_devices->seeding = 0;
1303
1304         return 0;
1305 }
1306
1307 int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1308 {
1309         struct btrfs_fs_devices *seed_devices = NULL;
1310         int ret;
1311
1312         mutex_lock(&uuid_mutex);
1313         ret = close_fs_devices(fs_devices);
1314         if (!fs_devices->opened) {
1315                 seed_devices = fs_devices->seed;
1316                 fs_devices->seed = NULL;
1317         }
1318         mutex_unlock(&uuid_mutex);
1319
1320         while (seed_devices) {
1321                 fs_devices = seed_devices;
1322                 seed_devices = fs_devices->seed;
1323                 close_fs_devices(fs_devices);
1324                 free_fs_devices(fs_devices);
1325         }
1326         return ret;
1327 }
1328
1329 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1330                                 fmode_t flags, void *holder)
1331 {
1332         struct btrfs_device *device;
1333         struct btrfs_device *latest_dev = NULL;
1334         int ret = 0;
1335
1336         flags |= FMODE_EXCL;
1337
1338         list_for_each_entry(device, &fs_devices->devices, dev_list) {
1339                 /* Just open everything we can; ignore failures here */
1340                 if (btrfs_open_one_device(fs_devices, device, flags, holder))
1341                         continue;
1342
1343                 if (!latest_dev ||
1344                     device->generation > latest_dev->generation)
1345                         latest_dev = device;
1346         }
1347         if (fs_devices->open_devices == 0) {
1348                 ret = -EINVAL;
1349                 goto out;
1350         }
1351         fs_devices->opened = 1;
1352         fs_devices->latest_bdev = latest_dev->bdev;
1353         fs_devices->total_rw_bytes = 0;
1354 out:
1355         return ret;
1356 }
1357
1358 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
1359 {
1360         struct btrfs_device *dev1, *dev2;
1361
1362         dev1 = list_entry(a, struct btrfs_device, dev_list);
1363         dev2 = list_entry(b, struct btrfs_device, dev_list);
1364
1365         if (dev1->devid < dev2->devid)
1366                 return -1;
1367         else if (dev1->devid > dev2->devid)
1368                 return 1;
1369         return 0;
1370 }
1371
1372 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1373                        fmode_t flags, void *holder)
1374 {
1375         int ret;
1376
1377         lockdep_assert_held(&uuid_mutex);
1378
1379         mutex_lock(&fs_devices->device_list_mutex);
1380         if (fs_devices->opened) {
1381                 fs_devices->opened++;
1382                 ret = 0;
1383         } else {
1384                 list_sort(NULL, &fs_devices->devices, devid_cmp);
1385                 ret = open_fs_devices(fs_devices, flags, holder);
1386         }
1387         mutex_unlock(&fs_devices->device_list_mutex);
1388
1389         return ret;
1390 }
1391
1392 static void btrfs_release_disk_super(struct page *page)
1393 {
1394         kunmap(page);
1395         put_page(page);
1396 }
1397
1398 static int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr,
1399                                  struct page **page,
1400                                  struct btrfs_super_block **disk_super)
1401 {
1402         void *p;
1403         pgoff_t index;
1404
1405         /* make sure our super fits in the device */
1406         if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1407                 return 1;
1408
1409         /* make sure our super fits in the page */
1410         if (sizeof(**disk_super) > PAGE_SIZE)
1411                 return 1;
1412
1413         /* make sure our super doesn't straddle pages on disk */
1414         index = bytenr >> PAGE_SHIFT;
1415         if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index)
1416                 return 1;
1417
1418         /* pull in the page with our super */
1419         *page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
1420                                    index, GFP_KERNEL);
1421
1422         if (IS_ERR_OR_NULL(*page))
1423                 return 1;
1424
1425         p = kmap(*page);
1426
1427         /* align our pointer to the offset of the super block */
1428         *disk_super = p + offset_in_page(bytenr);
1429
1430         if (btrfs_super_bytenr(*disk_super) != bytenr ||
1431             btrfs_super_magic(*disk_super) != BTRFS_MAGIC) {
1432                 btrfs_release_disk_super(*page);
1433                 return 1;
1434         }
1435
1436         if ((*disk_super)->label[0] &&
1437                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1])
1438                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0';
1439
1440         return 0;
1441 }
1442
1443 int btrfs_forget_devices(const char *path)
1444 {
1445         int ret;
1446
1447         mutex_lock(&uuid_mutex);
1448         ret = btrfs_free_stale_devices(strlen(path) ? path : NULL, NULL);
1449         mutex_unlock(&uuid_mutex);
1450
1451         return ret;
1452 }
1453
1454 /*
1455  * Look for a btrfs signature on a device. This may be called out of the mount path
1456  * and we are not allowed to call set_blocksize during the scan. The superblock
1457  * is read via pagecache
1458  */
1459 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1460                                            void *holder)
1461 {
1462         struct btrfs_super_block *disk_super;
1463         bool new_device_added = false;
1464         struct btrfs_device *device = NULL;
1465         struct block_device *bdev;
1466         struct page *page;
1467         u64 bytenr;
1468
1469         lockdep_assert_held(&uuid_mutex);
1470
1471         /*
1472          * we would like to check all the supers, but that would make
1473          * a btrfs mount succeed after a mkfs from a different FS.
1474          * So, we need to add a special mount option to scan for
1475          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1476          */
1477         bytenr = btrfs_sb_offset(0);
1478         flags |= FMODE_EXCL;
1479
1480         bdev = blkdev_get_by_path(path, flags, holder);
1481         if (IS_ERR(bdev))
1482                 return ERR_CAST(bdev);
1483
1484         if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super)) {
1485                 device = ERR_PTR(-EINVAL);
1486                 goto error_bdev_put;
1487         }
1488
1489         device = device_list_add(path, disk_super, &new_device_added);
1490         if (!IS_ERR(device)) {
1491                 if (new_device_added)
1492                         btrfs_free_stale_devices(path, device);
1493         }
1494
1495         btrfs_release_disk_super(page);
1496
1497 error_bdev_put:
1498         blkdev_put(bdev, flags);
1499
1500         return device;
1501 }
1502
1503 /*
1504  * Try to find a chunk that intersects [start, start + len] range and when one
1505  * such is found, record the end of it in *start
1506  */
1507 static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1508                                     u64 len)
1509 {
1510         u64 physical_start, physical_end;
1511
1512         lockdep_assert_held(&device->fs_info->chunk_mutex);
1513
1514         if (!find_first_extent_bit(&device->alloc_state, *start,
1515                                    &physical_start, &physical_end,
1516                                    CHUNK_ALLOCATED, NULL)) {
1517
1518                 if (in_range(physical_start, *start, len) ||
1519                     in_range(*start, physical_start,
1520                              physical_end - physical_start)) {
1521                         *start = physical_end + 1;
1522                         return true;
1523                 }
1524         }
1525         return false;
1526 }
1527
1528
1529 /*
1530  * find_free_dev_extent_start - find free space in the specified device
1531  * @device:       the device which we search the free space in
1532  * @num_bytes:    the size of the free space that we need
1533  * @search_start: the position from which to begin the search
1534  * @start:        store the start of the free space.
1535  * @len:          the size of the free space. that we find, or the size
1536  *                of the max free space if we don't find suitable free space
1537  *
1538  * this uses a pretty simple search, the expectation is that it is
1539  * called very infrequently and that a given device has a small number
1540  * of extents
1541  *
1542  * @start is used to store the start of the free space if we find. But if we
1543  * don't find suitable free space, it will be used to store the start position
1544  * of the max free space.
1545  *
1546  * @len is used to store the size of the free space that we find.
1547  * But if we don't find suitable free space, it is used to store the size of
1548  * the max free space.
1549  */
1550 int find_free_dev_extent_start(struct btrfs_device *device, u64 num_bytes,
1551                                u64 search_start, u64 *start, u64 *len)
1552 {
1553         struct btrfs_fs_info *fs_info = device->fs_info;
1554         struct btrfs_root *root = fs_info->dev_root;
1555         struct btrfs_key key;
1556         struct btrfs_dev_extent *dev_extent;
1557         struct btrfs_path *path;
1558         u64 hole_size;
1559         u64 max_hole_start;
1560         u64 max_hole_size;
1561         u64 extent_end;
1562         u64 search_end = device->total_bytes;
1563         int ret;
1564         int slot;
1565         struct extent_buffer *l;
1566
1567         /*
1568          * We don't want to overwrite the superblock on the drive nor any area
1569          * used by the boot loader (grub for example), so we make sure to start
1570          * at an offset of at least 1MB.
1571          */
1572         search_start = max_t(u64, search_start, SZ_1M);
1573
1574         path = btrfs_alloc_path();
1575         if (!path)
1576                 return -ENOMEM;
1577
1578         max_hole_start = search_start;
1579         max_hole_size = 0;
1580
1581 again:
1582         if (search_start >= search_end ||
1583                 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1584                 ret = -ENOSPC;
1585                 goto out;
1586         }
1587
1588         path->reada = READA_FORWARD;
1589         path->search_commit_root = 1;
1590         path->skip_locking = 1;
1591
1592         key.objectid = device->devid;
1593         key.offset = search_start;
1594         key.type = BTRFS_DEV_EXTENT_KEY;
1595
1596         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1597         if (ret < 0)
1598                 goto out;
1599         if (ret > 0) {
1600                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1601                 if (ret < 0)
1602                         goto out;
1603         }
1604
1605         while (1) {
1606                 l = path->nodes[0];
1607                 slot = path->slots[0];
1608                 if (slot >= btrfs_header_nritems(l)) {
1609                         ret = btrfs_next_leaf(root, path);
1610                         if (ret == 0)
1611                                 continue;
1612                         if (ret < 0)
1613                                 goto out;
1614
1615                         break;
1616                 }
1617                 btrfs_item_key_to_cpu(l, &key, slot);
1618
1619                 if (key.objectid < device->devid)
1620                         goto next;
1621
1622                 if (key.objectid > device->devid)
1623                         break;
1624
1625                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1626                         goto next;
1627
1628                 if (key.offset > search_start) {
1629                         hole_size = key.offset - search_start;
1630
1631                         /*
1632                          * Have to check before we set max_hole_start, otherwise
1633                          * we could end up sending back this offset anyway.
1634                          */
1635                         if (contains_pending_extent(device, &search_start,
1636                                                     hole_size)) {
1637                                 if (key.offset >= search_start)
1638                                         hole_size = key.offset - search_start;
1639                                 else
1640                                         hole_size = 0;
1641                         }
1642
1643                         if (hole_size > max_hole_size) {
1644                                 max_hole_start = search_start;
1645                                 max_hole_size = hole_size;
1646                         }
1647
1648                         /*
1649                          * If this free space is greater than which we need,
1650                          * it must be the max free space that we have found
1651                          * until now, so max_hole_start must point to the start
1652                          * of this free space and the length of this free space
1653                          * is stored in max_hole_size. Thus, we return
1654                          * max_hole_start and max_hole_size and go back to the
1655                          * caller.
1656                          */
1657                         if (hole_size >= num_bytes) {
1658                                 ret = 0;
1659                                 goto out;
1660                         }
1661                 }
1662
1663                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1664                 extent_end = key.offset + btrfs_dev_extent_length(l,
1665                                                                   dev_extent);
1666                 if (extent_end > search_start)
1667                         search_start = extent_end;
1668 next:
1669                 path->slots[0]++;
1670                 cond_resched();
1671         }
1672
1673         /*
1674          * At this point, search_start should be the end of
1675          * allocated dev extents, and when shrinking the device,
1676          * search_end may be smaller than search_start.
1677          */
1678         if (search_end > search_start) {
1679                 hole_size = search_end - search_start;
1680
1681                 if (contains_pending_extent(device, &search_start, hole_size)) {
1682                         btrfs_release_path(path);
1683                         goto again;
1684                 }
1685
1686                 if (hole_size > max_hole_size) {
1687                         max_hole_start = search_start;
1688                         max_hole_size = hole_size;
1689                 }
1690         }
1691
1692         /* See above. */
1693         if (max_hole_size < num_bytes)
1694                 ret = -ENOSPC;
1695         else
1696                 ret = 0;
1697
1698 out:
1699         btrfs_free_path(path);
1700         *start = max_hole_start;
1701         if (len)
1702                 *len = max_hole_size;
1703         return ret;
1704 }
1705
1706 int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1707                          u64 *start, u64 *len)
1708 {
1709         /* FIXME use last free of some kind */
1710         return find_free_dev_extent_start(device, num_bytes, 0, start, len);
1711 }
1712
1713 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1714                           struct btrfs_device *device,
1715                           u64 start, u64 *dev_extent_len)
1716 {
1717         struct btrfs_fs_info *fs_info = device->fs_info;
1718         struct btrfs_root *root = fs_info->dev_root;
1719         int ret;
1720         struct btrfs_path *path;
1721         struct btrfs_key key;
1722         struct btrfs_key found_key;
1723         struct extent_buffer *leaf = NULL;
1724         struct btrfs_dev_extent *extent = NULL;
1725
1726         path = btrfs_alloc_path();
1727         if (!path)
1728                 return -ENOMEM;
1729
1730         key.objectid = device->devid;
1731         key.offset = start;
1732         key.type = BTRFS_DEV_EXTENT_KEY;
1733 again:
1734         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1735         if (ret > 0) {
1736                 ret = btrfs_previous_item(root, path, key.objectid,
1737                                           BTRFS_DEV_EXTENT_KEY);
1738                 if (ret)
1739                         goto out;
1740                 leaf = path->nodes[0];
1741                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1742                 extent = btrfs_item_ptr(leaf, path->slots[0],
1743                                         struct btrfs_dev_extent);
1744                 BUG_ON(found_key.offset > start || found_key.offset +
1745                        btrfs_dev_extent_length(leaf, extent) < start);
1746                 key = found_key;
1747                 btrfs_release_path(path);
1748                 goto again;
1749         } else if (ret == 0) {
1750                 leaf = path->nodes[0];
1751                 extent = btrfs_item_ptr(leaf, path->slots[0],
1752                                         struct btrfs_dev_extent);
1753         } else {
1754                 btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1755                 goto out;
1756         }
1757
1758         *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1759
1760         ret = btrfs_del_item(trans, root, path);
1761         if (ret) {
1762                 btrfs_handle_fs_error(fs_info, ret,
1763                                       "Failed to remove dev extent item");
1764         } else {
1765                 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1766         }
1767 out:
1768         btrfs_free_path(path);
1769         return ret;
1770 }
1771
1772 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1773                                   struct btrfs_device *device,
1774                                   u64 chunk_offset, u64 start, u64 num_bytes)
1775 {
1776         int ret;
1777         struct btrfs_path *path;
1778         struct btrfs_fs_info *fs_info = device->fs_info;
1779         struct btrfs_root *root = fs_info->dev_root;
1780         struct btrfs_dev_extent *extent;
1781         struct extent_buffer *leaf;
1782         struct btrfs_key key;
1783
1784         WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state));
1785         WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1786         path = btrfs_alloc_path();
1787         if (!path)
1788                 return -ENOMEM;
1789
1790         key.objectid = device->devid;
1791         key.offset = start;
1792         key.type = BTRFS_DEV_EXTENT_KEY;
1793         ret = btrfs_insert_empty_item(trans, root, path, &key,
1794                                       sizeof(*extent));
1795         if (ret)
1796                 goto out;
1797
1798         leaf = path->nodes[0];
1799         extent = btrfs_item_ptr(leaf, path->slots[0],
1800                                 struct btrfs_dev_extent);
1801         btrfs_set_dev_extent_chunk_tree(leaf, extent,
1802                                         BTRFS_CHUNK_TREE_OBJECTID);
1803         btrfs_set_dev_extent_chunk_objectid(leaf, extent,
1804                                             BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1805         btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1806
1807         btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1808         btrfs_mark_buffer_dirty(leaf);
1809 out:
1810         btrfs_free_path(path);
1811         return ret;
1812 }
1813
1814 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1815 {
1816         struct extent_map_tree *em_tree;
1817         struct extent_map *em;
1818         struct rb_node *n;
1819         u64 ret = 0;
1820
1821         em_tree = &fs_info->mapping_tree.map_tree;
1822         read_lock(&em_tree->lock);
1823         n = rb_last(&em_tree->map.rb_root);
1824         if (n) {
1825                 em = rb_entry(n, struct extent_map, rb_node);
1826                 ret = em->start + em->len;
1827         }
1828         read_unlock(&em_tree->lock);
1829
1830         return ret;
1831 }
1832
1833 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1834                                     u64 *devid_ret)
1835 {
1836         int ret;
1837         struct btrfs_key key;
1838         struct btrfs_key found_key;
1839         struct btrfs_path *path;
1840
1841         path = btrfs_alloc_path();
1842         if (!path)
1843                 return -ENOMEM;
1844
1845         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1846         key.type = BTRFS_DEV_ITEM_KEY;
1847         key.offset = (u64)-1;
1848
1849         ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1850         if (ret < 0)
1851                 goto error;
1852
1853         BUG_ON(ret == 0); /* Corruption */
1854
1855         ret = btrfs_previous_item(fs_info->chunk_root, path,
1856                                   BTRFS_DEV_ITEMS_OBJECTID,
1857                                   BTRFS_DEV_ITEM_KEY);
1858         if (ret) {
1859                 *devid_ret = 1;
1860         } else {
1861                 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1862                                       path->slots[0]);
1863                 *devid_ret = found_key.offset + 1;
1864         }
1865         ret = 0;
1866 error:
1867         btrfs_free_path(path);
1868         return ret;
1869 }
1870
1871 /*
1872  * the device information is stored in the chunk root
1873  * the btrfs_device struct should be fully filled in
1874  */
1875 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1876                             struct btrfs_device *device)
1877 {
1878         int ret;
1879         struct btrfs_path *path;
1880         struct btrfs_dev_item *dev_item;
1881         struct extent_buffer *leaf;
1882         struct btrfs_key key;
1883         unsigned long ptr;
1884
1885         path = btrfs_alloc_path();
1886         if (!path)
1887                 return -ENOMEM;
1888
1889         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1890         key.type = BTRFS_DEV_ITEM_KEY;
1891         key.offset = device->devid;
1892
1893         ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1894                                       &key, sizeof(*dev_item));
1895         if (ret)
1896                 goto out;
1897
1898         leaf = path->nodes[0];
1899         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1900
1901         btrfs_set_device_id(leaf, dev_item, device->devid);
1902         btrfs_set_device_generation(leaf, dev_item, 0);
1903         btrfs_set_device_type(leaf, dev_item, device->type);
1904         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1905         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1906         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1907         btrfs_set_device_total_bytes(leaf, dev_item,
1908                                      btrfs_device_get_disk_total_bytes(device));
1909         btrfs_set_device_bytes_used(leaf, dev_item,
1910                                     btrfs_device_get_bytes_used(device));
1911         btrfs_set_device_group(leaf, dev_item, 0);
1912         btrfs_set_device_seek_speed(leaf, dev_item, 0);
1913         btrfs_set_device_bandwidth(leaf, dev_item, 0);
1914         btrfs_set_device_start_offset(leaf, dev_item, 0);
1915
1916         ptr = btrfs_device_uuid(dev_item);
1917         write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1918         ptr = btrfs_device_fsid(dev_item);
1919         write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1920                             ptr, BTRFS_FSID_SIZE);
1921         btrfs_mark_buffer_dirty(leaf);
1922
1923         ret = 0;
1924 out:
1925         btrfs_free_path(path);
1926         return ret;
1927 }
1928
1929 /*
1930  * Function to update ctime/mtime for a given device path.
1931  * Mainly used for ctime/mtime based probe like libblkid.
1932  */
1933 static void update_dev_time(const char *path_name)
1934 {
1935         struct file *filp;
1936
1937         filp = filp_open(path_name, O_RDWR, 0);
1938         if (IS_ERR(filp))
1939                 return;
1940         file_update_time(filp);
1941         filp_close(filp, NULL);
1942 }
1943
1944 static int btrfs_rm_dev_item(struct btrfs_device *device)
1945 {
1946         struct btrfs_root *root = device->fs_info->chunk_root;
1947         int ret;
1948         struct btrfs_path *path;
1949         struct btrfs_key key;
1950         struct btrfs_trans_handle *trans;
1951
1952         path = btrfs_alloc_path();
1953         if (!path)
1954                 return -ENOMEM;
1955
1956         trans = btrfs_start_transaction(root, 0);
1957         if (IS_ERR(trans)) {
1958                 btrfs_free_path(path);
1959                 return PTR_ERR(trans);
1960         }
1961         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1962         key.type = BTRFS_DEV_ITEM_KEY;
1963         key.offset = device->devid;
1964
1965         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1966         if (ret) {
1967                 if (ret > 0)
1968                         ret = -ENOENT;
1969                 btrfs_abort_transaction(trans, ret);
1970                 btrfs_end_transaction(trans);
1971                 goto out;
1972         }
1973
1974         ret = btrfs_del_item(trans, root, path);
1975         if (ret) {
1976                 btrfs_abort_transaction(trans, ret);
1977                 btrfs_end_transaction(trans);
1978         }
1979
1980 out:
1981         btrfs_free_path(path);
1982         if (!ret)
1983                 ret = btrfs_commit_transaction(trans);
1984         return ret;
1985 }
1986
1987 /*
1988  * Verify that @num_devices satisfies the RAID profile constraints in the whole
1989  * filesystem. It's up to the caller to adjust that number regarding eg. device
1990  * replace.
1991  */
1992 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1993                 u64 num_devices)
1994 {
1995         u64 all_avail;
1996         unsigned seq;
1997         int i;
1998
1999         do {
2000                 seq = read_seqbegin(&fs_info->profiles_lock);
2001
2002                 all_avail = fs_info->avail_data_alloc_bits |
2003                             fs_info->avail_system_alloc_bits |
2004                             fs_info->avail_metadata_alloc_bits;
2005         } while (read_seqretry(&fs_info->profiles_lock, seq));
2006
2007         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2008                 if (!(all_avail & btrfs_raid_array[i].bg_flag))
2009                         continue;
2010
2011                 if (num_devices < btrfs_raid_array[i].devs_min) {
2012                         int ret = btrfs_raid_array[i].mindev_error;
2013
2014                         if (ret)
2015                                 return ret;
2016                 }
2017         }
2018
2019         return 0;
2020 }
2021
2022 static struct btrfs_device * btrfs_find_next_active_device(
2023                 struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2024 {
2025         struct btrfs_device *next_device;
2026
2027         list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2028                 if (next_device != device &&
2029                     !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2030                     && next_device->bdev)
2031                         return next_device;
2032         }
2033
2034         return NULL;
2035 }
2036
2037 /*
2038  * Helper function to check if the given device is part of s_bdev / latest_bdev
2039  * and replace it with the provided or the next active device, in the context
2040  * where this function called, there should be always be another device (or
2041  * this_dev) which is active.
2042  */
2043 void btrfs_assign_next_active_device(struct btrfs_device *device,
2044                                      struct btrfs_device *this_dev)
2045 {
2046         struct btrfs_fs_info *fs_info = device->fs_info;
2047         struct btrfs_device *next_device;
2048
2049         if (this_dev)
2050                 next_device = this_dev;
2051         else
2052                 next_device = btrfs_find_next_active_device(fs_info->fs_devices,
2053                                                                 device);
2054         ASSERT(next_device);
2055
2056         if (fs_info->sb->s_bdev &&
2057                         (fs_info->sb->s_bdev == device->bdev))
2058                 fs_info->sb->s_bdev = next_device->bdev;
2059
2060         if (fs_info->fs_devices->latest_bdev == device->bdev)
2061                 fs_info->fs_devices->latest_bdev = next_device->bdev;
2062 }
2063
2064 /*
2065  * Return btrfs_fs_devices::num_devices excluding the device that's being
2066  * currently replaced.
2067  */
2068 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2069 {
2070         u64 num_devices = fs_info->fs_devices->num_devices;
2071
2072         down_read(&fs_info->dev_replace.rwsem);
2073         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
2074                 ASSERT(num_devices > 1);
2075                 num_devices--;
2076         }
2077         up_read(&fs_info->dev_replace.rwsem);
2078
2079         return num_devices;
2080 }
2081
2082 int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
2083                 u64 devid)
2084 {
2085         struct btrfs_device *device;
2086         struct btrfs_fs_devices *cur_devices;
2087         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2088         u64 num_devices;
2089         int ret = 0;
2090
2091         mutex_lock(&uuid_mutex);
2092
2093         num_devices = btrfs_num_devices(fs_info);
2094
2095         ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2096         if (ret)
2097                 goto out;
2098
2099         device = btrfs_find_device_by_devspec(fs_info, devid, device_path);
2100
2101         if (IS_ERR(device)) {
2102                 if (PTR_ERR(device) == -ENOENT &&
2103                     strcmp(device_path, "missing") == 0)
2104                         ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2105                 else
2106                         ret = PTR_ERR(device);
2107                 goto out;
2108         }
2109
2110         if (btrfs_pinned_by_swapfile(fs_info, device)) {
2111                 btrfs_warn_in_rcu(fs_info,
2112                   "cannot remove device %s (devid %llu) due to active swapfile",
2113                                   rcu_str_deref(device->name), device->devid);
2114                 ret = -ETXTBSY;
2115                 goto out;
2116         }
2117
2118         if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2119                 ret = BTRFS_ERROR_DEV_TGT_REPLACE;
2120                 goto out;
2121         }
2122
2123         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2124             fs_info->fs_devices->rw_devices == 1) {
2125                 ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
2126                 goto out;
2127         }
2128
2129         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2130                 mutex_lock(&fs_info->chunk_mutex);
2131                 list_del_init(&device->dev_alloc_list);
2132                 device->fs_devices->rw_devices--;
2133                 mutex_unlock(&fs_info->chunk_mutex);
2134         }
2135
2136         mutex_unlock(&uuid_mutex);
2137         ret = btrfs_shrink_device(device, 0);
2138         mutex_lock(&uuid_mutex);
2139         if (ret)
2140                 goto error_undo;
2141
2142         /*
2143          * TODO: the superblock still includes this device in its num_devices
2144          * counter although write_all_supers() is not locked out. This
2145          * could give a filesystem state which requires a degraded mount.
2146          */
2147         ret = btrfs_rm_dev_item(device);
2148         if (ret)
2149                 goto error_undo;
2150
2151         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2152         btrfs_scrub_cancel_dev(device);
2153
2154         /*
2155          * the device list mutex makes sure that we don't change
2156          * the device list while someone else is writing out all
2157          * the device supers. Whoever is writing all supers, should
2158          * lock the device list mutex before getting the number of
2159          * devices in the super block (super_copy). Conversely,
2160          * whoever updates the number of devices in the super block
2161          * (super_copy) should hold the device list mutex.
2162          */
2163
2164         /*
2165          * In normal cases the cur_devices == fs_devices. But in case
2166          * of deleting a seed device, the cur_devices should point to
2167          * its own fs_devices listed under the fs_devices->seed.
2168          */
2169         cur_devices = device->fs_devices;
2170         mutex_lock(&fs_devices->device_list_mutex);
2171         list_del_rcu(&device->dev_list);
2172
2173         cur_devices->num_devices--;
2174         cur_devices->total_devices--;
2175         /* Update total_devices of the parent fs_devices if it's seed */
2176         if (cur_devices != fs_devices)
2177                 fs_devices->total_devices--;
2178
2179         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2180                 cur_devices->missing_devices--;
2181
2182         btrfs_assign_next_active_device(device, NULL);
2183
2184         if (device->bdev) {
2185                 cur_devices->open_devices--;
2186                 /* remove sysfs entry */
2187                 btrfs_sysfs_rm_device_link(fs_devices, device);
2188         }
2189
2190         num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2191         btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2192         mutex_unlock(&fs_devices->device_list_mutex);
2193
2194         /*
2195          * at this point, the device is zero sized and detached from
2196          * the devices list.  All that's left is to zero out the old
2197          * supers and free the device.
2198          */
2199         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2200                 btrfs_scratch_superblocks(device->bdev, device->name->str);
2201
2202         btrfs_close_bdev(device);
2203         synchronize_rcu();
2204         btrfs_free_device(device);
2205
2206         if (cur_devices->open_devices == 0) {
2207                 while (fs_devices) {
2208                         if (fs_devices->seed == cur_devices) {
2209                                 fs_devices->seed = cur_devices->seed;
2210                                 break;
2211                         }
2212                         fs_devices = fs_devices->seed;
2213                 }
2214                 cur_devices->seed = NULL;
2215                 close_fs_devices(cur_devices);
2216                 free_fs_devices(cur_devices);
2217         }
2218
2219 out:
2220         mutex_unlock(&uuid_mutex);
2221         return ret;
2222
2223 error_undo:
2224         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2225                 mutex_lock(&fs_info->chunk_mutex);
2226                 list_add(&device->dev_alloc_list,
2227                          &fs_devices->alloc_list);
2228                 device->fs_devices->rw_devices++;
2229                 mutex_unlock(&fs_info->chunk_mutex);
2230         }
2231         goto out;
2232 }
2233
2234 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2235 {
2236         struct btrfs_fs_devices *fs_devices;
2237
2238         lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2239
2240         /*
2241          * in case of fs with no seed, srcdev->fs_devices will point
2242          * to fs_devices of fs_info. However when the dev being replaced is
2243          * a seed dev it will point to the seed's local fs_devices. In short
2244          * srcdev will have its correct fs_devices in both the cases.
2245          */
2246         fs_devices = srcdev->fs_devices;
2247
2248         list_del_rcu(&srcdev->dev_list);
2249         list_del(&srcdev->dev_alloc_list);
2250         fs_devices->num_devices--;
2251         if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2252                 fs_devices->missing_devices--;
2253
2254         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2255                 fs_devices->rw_devices--;
2256
2257         if (srcdev->bdev)
2258                 fs_devices->open_devices--;
2259 }
2260
2261 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2262 {
2263         struct btrfs_fs_info *fs_info = srcdev->fs_info;
2264         struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2265
2266         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) {
2267                 /* zero out the old super if it is writable */
2268                 btrfs_scratch_superblocks(srcdev->bdev, srcdev->name->str);
2269         }
2270
2271         btrfs_close_bdev(srcdev);
2272         synchronize_rcu();
2273         btrfs_free_device(srcdev);
2274
2275         /* if this is no devs we rather delete the fs_devices */
2276         if (!fs_devices->num_devices) {
2277                 struct btrfs_fs_devices *tmp_fs_devices;
2278
2279                 /*
2280                  * On a mounted FS, num_devices can't be zero unless it's a
2281                  * seed. In case of a seed device being replaced, the replace
2282                  * target added to the sprout FS, so there will be no more
2283                  * device left under the seed FS.
2284                  */
2285                 ASSERT(fs_devices->seeding);
2286
2287                 tmp_fs_devices = fs_info->fs_devices;
2288                 while (tmp_fs_devices) {
2289                         if (tmp_fs_devices->seed == fs_devices) {
2290                                 tmp_fs_devices->seed = fs_devices->seed;
2291                                 break;
2292                         }
2293                         tmp_fs_devices = tmp_fs_devices->seed;
2294                 }
2295                 fs_devices->seed = NULL;
2296                 close_fs_devices(fs_devices);
2297                 free_fs_devices(fs_devices);
2298         }
2299 }
2300
2301 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2302 {
2303         struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2304
2305         WARN_ON(!tgtdev);
2306         mutex_lock(&fs_devices->device_list_mutex);
2307
2308         btrfs_sysfs_rm_device_link(fs_devices, tgtdev);
2309
2310         if (tgtdev->bdev)
2311                 fs_devices->open_devices--;
2312
2313         fs_devices->num_devices--;
2314
2315         btrfs_assign_next_active_device(tgtdev, NULL);
2316
2317         list_del_rcu(&tgtdev->dev_list);
2318
2319         mutex_unlock(&fs_devices->device_list_mutex);
2320
2321         /*
2322          * The update_dev_time() with in btrfs_scratch_superblocks()
2323          * may lead to a call to btrfs_show_devname() which will try
2324          * to hold device_list_mutex. And here this device
2325          * is already out of device list, so we don't have to hold
2326          * the device_list_mutex lock.
2327          */
2328         btrfs_scratch_superblocks(tgtdev->bdev, tgtdev->name->str);
2329
2330         btrfs_close_bdev(tgtdev);
2331         synchronize_rcu();
2332         btrfs_free_device(tgtdev);
2333 }
2334
2335 static struct btrfs_device *btrfs_find_device_by_path(
2336                 struct btrfs_fs_info *fs_info, const char *device_path)
2337 {
2338         int ret = 0;
2339         struct btrfs_super_block *disk_super;
2340         u64 devid;
2341         u8 *dev_uuid;
2342         struct block_device *bdev;
2343         struct buffer_head *bh;
2344         struct btrfs_device *device;
2345
2346         ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2347                                     fs_info->bdev_holder, 0, &bdev, &bh);
2348         if (ret)
2349                 return ERR_PTR(ret);
2350         disk_super = (struct btrfs_super_block *)bh->b_data;
2351         devid = btrfs_stack_device_id(&disk_super->dev_item);
2352         dev_uuid = disk_super->dev_item.uuid;
2353         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2354                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2355                                            disk_super->metadata_uuid, true);
2356         else
2357                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2358                                            disk_super->fsid, true);
2359
2360         brelse(bh);
2361         if (!device)
2362                 device = ERR_PTR(-ENOENT);
2363         blkdev_put(bdev, FMODE_READ);
2364         return device;
2365 }
2366
2367 /*
2368  * Lookup a device given by device id, or the path if the id is 0.
2369  */
2370 struct btrfs_device *btrfs_find_device_by_devspec(
2371                 struct btrfs_fs_info *fs_info, u64 devid,
2372                 const char *device_path)
2373 {
2374         struct btrfs_device *device;
2375
2376         if (devid) {
2377                 device = btrfs_find_device(fs_info->fs_devices, devid, NULL,
2378                                            NULL, true);
2379                 if (!device)
2380                         return ERR_PTR(-ENOENT);
2381                 return device;
2382         }
2383
2384         if (!device_path || !device_path[0])
2385                 return ERR_PTR(-EINVAL);
2386
2387         if (strcmp(device_path, "missing") == 0) {
2388                 /* Find first missing device */
2389                 list_for_each_entry(device, &fs_info->fs_devices->devices,
2390                                     dev_list) {
2391                         if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
2392                                      &device->dev_state) && !device->bdev)
2393                                 return device;
2394                 }
2395                 return ERR_PTR(-ENOENT);
2396         }
2397
2398         return btrfs_find_device_by_path(fs_info, device_path);
2399 }
2400
2401 /*
2402  * does all the dirty work required for changing file system's UUID.
2403  */
2404 static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
2405 {
2406         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2407         struct btrfs_fs_devices *old_devices;
2408         struct btrfs_fs_devices *seed_devices;
2409         struct btrfs_super_block *disk_super = fs_info->super_copy;
2410         struct btrfs_device *device;
2411         u64 super_flags;
2412
2413         lockdep_assert_held(&uuid_mutex);
2414         if (!fs_devices->seeding)
2415                 return -EINVAL;
2416
2417         seed_devices = alloc_fs_devices(NULL, NULL);
2418         if (IS_ERR(seed_devices))
2419                 return PTR_ERR(seed_devices);
2420
2421         old_devices = clone_fs_devices(fs_devices);
2422         if (IS_ERR(old_devices)) {
2423                 kfree(seed_devices);
2424                 return PTR_ERR(old_devices);
2425         }
2426
2427         list_add(&old_devices->fs_list, &fs_uuids);
2428
2429         memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2430         seed_devices->opened = 1;
2431         INIT_LIST_HEAD(&seed_devices->devices);
2432         INIT_LIST_HEAD(&seed_devices->alloc_list);
2433         mutex_init(&seed_devices->device_list_mutex);
2434
2435         mutex_lock(&fs_devices->device_list_mutex);
2436         list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2437                               synchronize_rcu);
2438         list_for_each_entry(device, &seed_devices->devices, dev_list)
2439                 device->fs_devices = seed_devices;
2440
2441         mutex_lock(&fs_info->chunk_mutex);
2442         list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
2443         mutex_unlock(&fs_info->chunk_mutex);
2444
2445         fs_devices->seeding = 0;
2446         fs_devices->num_devices = 0;
2447         fs_devices->open_devices = 0;
2448         fs_devices->missing_devices = 0;
2449         fs_devices->rotating = 0;
2450         fs_devices->seed = seed_devices;
2451
2452         generate_random_uuid(fs_devices->fsid);
2453         memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2454         memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2455         mutex_unlock(&fs_devices->device_list_mutex);
2456
2457         super_flags = btrfs_super_flags(disk_super) &
2458                       ~BTRFS_SUPER_FLAG_SEEDING;
2459         btrfs_set_super_flags(disk_super, super_flags);
2460
2461         return 0;
2462 }
2463
2464 /*
2465  * Store the expected generation for seed devices in device items.
2466  */
2467 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2468 {
2469         struct btrfs_fs_info *fs_info = trans->fs_info;
2470         struct btrfs_root *root = fs_info->chunk_root;
2471         struct btrfs_path *path;
2472         struct extent_buffer *leaf;
2473         struct btrfs_dev_item *dev_item;
2474         struct btrfs_device *device;
2475         struct btrfs_key key;
2476         u8 fs_uuid[BTRFS_FSID_SIZE];
2477         u8 dev_uuid[BTRFS_UUID_SIZE];
2478         u64 devid;
2479         int ret;
2480
2481         path = btrfs_alloc_path();
2482         if (!path)
2483                 return -ENOMEM;
2484
2485         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2486         key.offset = 0;
2487         key.type = BTRFS_DEV_ITEM_KEY;
2488
2489         while (1) {
2490                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2491                 if (ret < 0)
2492                         goto error;
2493
2494                 leaf = path->nodes[0];
2495 next_slot:
2496                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2497                         ret = btrfs_next_leaf(root, path);
2498                         if (ret > 0)
2499                                 break;
2500                         if (ret < 0)
2501                                 goto error;
2502                         leaf = path->nodes[0];
2503                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2504                         btrfs_release_path(path);
2505                         continue;
2506                 }
2507
2508                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2509                 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2510                     key.type != BTRFS_DEV_ITEM_KEY)
2511                         break;
2512
2513                 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2514                                           struct btrfs_dev_item);
2515                 devid = btrfs_device_id(leaf, dev_item);
2516                 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2517                                    BTRFS_UUID_SIZE);
2518                 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2519                                    BTRFS_FSID_SIZE);
2520                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2521                                            fs_uuid, true);
2522                 BUG_ON(!device); /* Logic error */
2523
2524                 if (device->fs_devices->seeding) {
2525                         btrfs_set_device_generation(leaf, dev_item,
2526                                                     device->generation);
2527                         btrfs_mark_buffer_dirty(leaf);
2528                 }
2529
2530                 path->slots[0]++;
2531                 goto next_slot;
2532         }
2533         ret = 0;
2534 error:
2535         btrfs_free_path(path);
2536         return ret;
2537 }
2538
2539 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2540 {
2541         struct btrfs_root *root = fs_info->dev_root;
2542         struct request_queue *q;
2543         struct btrfs_trans_handle *trans;
2544         struct btrfs_device *device;
2545         struct block_device *bdev;
2546         struct super_block *sb = fs_info->sb;
2547         struct rcu_string *name;
2548         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2549         u64 orig_super_total_bytes;
2550         u64 orig_super_num_devices;
2551         int seeding_dev = 0;
2552         int ret = 0;
2553         bool unlocked = false;
2554
2555         if (sb_rdonly(sb) && !fs_devices->seeding)
2556                 return -EROFS;
2557
2558         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2559                                   fs_info->bdev_holder);
2560         if (IS_ERR(bdev))
2561                 return PTR_ERR(bdev);
2562
2563         if (fs_devices->seeding) {
2564                 seeding_dev = 1;
2565                 down_write(&sb->s_umount);
2566                 mutex_lock(&uuid_mutex);
2567         }
2568
2569         filemap_write_and_wait(bdev->bd_inode->i_mapping);
2570
2571         mutex_lock(&fs_devices->device_list_mutex);
2572         list_for_each_entry(device, &fs_devices->devices, dev_list) {
2573                 if (device->bdev == bdev) {
2574                         ret = -EEXIST;
2575                         mutex_unlock(
2576                                 &fs_devices->device_list_mutex);
2577                         goto error;
2578                 }
2579         }
2580         mutex_unlock(&fs_devices->device_list_mutex);
2581
2582         device = btrfs_alloc_device(fs_info, NULL, NULL);
2583         if (IS_ERR(device)) {
2584                 /* we can safely leave the fs_devices entry around */
2585                 ret = PTR_ERR(device);
2586                 goto error;
2587         }
2588
2589         name = rcu_string_strdup(device_path, GFP_KERNEL);
2590         if (!name) {
2591                 ret = -ENOMEM;
2592                 goto error_free_device;
2593         }
2594         rcu_assign_pointer(device->name, name);
2595
2596         trans = btrfs_start_transaction(root, 0);
2597         if (IS_ERR(trans)) {
2598                 ret = PTR_ERR(trans);
2599                 goto error_free_device;
2600         }
2601
2602         q = bdev_get_queue(bdev);
2603         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2604         device->generation = trans->transid;
2605         device->io_width = fs_info->sectorsize;
2606         device->io_align = fs_info->sectorsize;
2607         device->sector_size = fs_info->sectorsize;
2608         device->total_bytes = round_down(i_size_read(bdev->bd_inode),
2609                                          fs_info->sectorsize);
2610         device->disk_total_bytes = device->total_bytes;
2611         device->commit_total_bytes = device->total_bytes;
2612         device->fs_info = fs_info;
2613         device->bdev = bdev;
2614         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2615         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2616         device->mode = FMODE_EXCL;
2617         device->dev_stats_valid = 1;
2618         set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2619
2620         if (seeding_dev) {
2621                 sb->s_flags &= ~SB_RDONLY;
2622                 ret = btrfs_prepare_sprout(fs_info);
2623                 if (ret) {
2624                         btrfs_abort_transaction(trans, ret);
2625                         goto error_trans;
2626                 }
2627         }
2628
2629         device->fs_devices = fs_devices;
2630
2631         mutex_lock(&fs_devices->device_list_mutex);
2632         mutex_lock(&fs_info->chunk_mutex);
2633         list_add_rcu(&device->dev_list, &fs_devices->devices);
2634         list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2635         fs_devices->num_devices++;
2636         fs_devices->open_devices++;
2637         fs_devices->rw_devices++;
2638         fs_devices->total_devices++;
2639         fs_devices->total_rw_bytes += device->total_bytes;
2640
2641         atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2642
2643         if (!blk_queue_nonrot(q))
2644                 fs_devices->rotating = 1;
2645
2646         orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2647         btrfs_set_super_total_bytes(fs_info->super_copy,
2648                 round_down(orig_super_total_bytes + device->total_bytes,
2649                            fs_info->sectorsize));
2650
2651         orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2652         btrfs_set_super_num_devices(fs_info->super_copy,
2653                                     orig_super_num_devices + 1);
2654
2655         /* add sysfs device entry */
2656         btrfs_sysfs_add_device_link(fs_devices, device);
2657
2658         /*
2659          * we've got more storage, clear any full flags on the space
2660          * infos
2661          */
2662         btrfs_clear_space_info_full(fs_info);
2663
2664         mutex_unlock(&fs_info->chunk_mutex);
2665         mutex_unlock(&fs_devices->device_list_mutex);
2666
2667         if (seeding_dev) {
2668                 mutex_lock(&fs_info->chunk_mutex);
2669                 ret = init_first_rw_device(trans);
2670                 mutex_unlock(&fs_info->chunk_mutex);
2671                 if (ret) {
2672                         btrfs_abort_transaction(trans, ret);
2673                         goto error_sysfs;
2674                 }
2675         }
2676
2677         ret = btrfs_add_dev_item(trans, device);
2678         if (ret) {
2679                 btrfs_abort_transaction(trans, ret);
2680                 goto error_sysfs;
2681         }
2682
2683         if (seeding_dev) {
2684                 char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
2685
2686                 ret = btrfs_finish_sprout(trans);
2687                 if (ret) {
2688                         btrfs_abort_transaction(trans, ret);
2689                         goto error_sysfs;
2690                 }
2691
2692                 /* Sprouting would change fsid of the mounted root,
2693                  * so rename the fsid on the sysfs
2694                  */
2695                 snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
2696                                                 fs_info->fs_devices->fsid);
2697                 if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf))
2698                         btrfs_warn(fs_info,
2699                                    "sysfs: failed to create fsid for sprout");
2700         }
2701
2702         ret = btrfs_commit_transaction(trans);
2703
2704         if (seeding_dev) {
2705                 mutex_unlock(&uuid_mutex);
2706                 up_write(&sb->s_umount);
2707                 unlocked = true;
2708
2709                 if (ret) /* transaction commit */
2710                         return ret;
2711
2712                 ret = btrfs_relocate_sys_chunks(fs_info);
2713                 if (ret < 0)
2714                         btrfs_handle_fs_error(fs_info, ret,
2715                                     "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2716                 trans = btrfs_attach_transaction(root);
2717                 if (IS_ERR(trans)) {
2718                         if (PTR_ERR(trans) == -ENOENT)
2719                                 return 0;
2720                         ret = PTR_ERR(trans);
2721                         trans = NULL;
2722                         goto error_sysfs;
2723                 }
2724                 ret = btrfs_commit_transaction(trans);
2725         }
2726
2727         /* Update ctime/mtime for libblkid */
2728         update_dev_time(device_path);
2729         return ret;
2730
2731 error_sysfs:
2732         btrfs_sysfs_rm_device_link(fs_devices, device);
2733         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2734         mutex_lock(&fs_info->chunk_mutex);
2735         list_del_rcu(&device->dev_list);
2736         list_del(&device->dev_alloc_list);
2737         fs_info->fs_devices->num_devices--;
2738         fs_info->fs_devices->open_devices--;
2739         fs_info->fs_devices->rw_devices--;
2740         fs_info->fs_devices->total_devices--;
2741         fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2742         atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2743         btrfs_set_super_total_bytes(fs_info->super_copy,
2744                                     orig_super_total_bytes);
2745         btrfs_set_super_num_devices(fs_info->super_copy,
2746                                     orig_super_num_devices);
2747         mutex_unlock(&fs_info->chunk_mutex);
2748         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2749 error_trans:
2750         if (seeding_dev)
2751                 sb->s_flags |= SB_RDONLY;
2752         if (trans)
2753                 btrfs_end_transaction(trans);
2754 error_free_device:
2755         btrfs_free_device(device);
2756 error:
2757         blkdev_put(bdev, FMODE_EXCL);
2758         if (seeding_dev && !unlocked) {
2759                 mutex_unlock(&uuid_mutex);
2760                 up_write(&sb->s_umount);
2761         }
2762         return ret;
2763 }
2764
2765 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2766                                         struct btrfs_device *device)
2767 {
2768         int ret;
2769         struct btrfs_path *path;
2770         struct btrfs_root *root = device->fs_info->chunk_root;
2771         struct btrfs_dev_item *dev_item;
2772         struct extent_buffer *leaf;
2773         struct btrfs_key key;
2774
2775         path = btrfs_alloc_path();
2776         if (!path)
2777                 return -ENOMEM;
2778
2779         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2780         key.type = BTRFS_DEV_ITEM_KEY;
2781         key.offset = device->devid;
2782
2783         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2784         if (ret < 0)
2785                 goto out;
2786
2787         if (ret > 0) {
2788                 ret = -ENOENT;
2789                 goto out;
2790         }
2791
2792         leaf = path->nodes[0];
2793         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2794
2795         btrfs_set_device_id(leaf, dev_item, device->devid);
2796         btrfs_set_device_type(leaf, dev_item, device->type);
2797         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2798         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2799         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2800         btrfs_set_device_total_bytes(leaf, dev_item,
2801                                      btrfs_device_get_disk_total_bytes(device));
2802         btrfs_set_device_bytes_used(leaf, dev_item,
2803                                     btrfs_device_get_bytes_used(device));
2804         btrfs_mark_buffer_dirty(leaf);
2805
2806 out:
2807         btrfs_free_path(path);
2808         return ret;
2809 }
2810
2811 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2812                       struct btrfs_device *device, u64 new_size)
2813 {
2814         struct btrfs_fs_info *fs_info = device->fs_info;
2815         struct btrfs_super_block *super_copy = fs_info->super_copy;
2816         u64 old_total;
2817         u64 diff;
2818
2819         if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2820                 return -EACCES;
2821
2822         new_size = round_down(new_size, fs_info->sectorsize);
2823
2824         mutex_lock(&fs_info->chunk_mutex);
2825         old_total = btrfs_super_total_bytes(super_copy);
2826         diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2827
2828         if (new_size <= device->total_bytes ||
2829             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2830                 mutex_unlock(&fs_info->chunk_mutex);
2831                 return -EINVAL;
2832         }
2833
2834         btrfs_set_super_total_bytes(super_copy,
2835                         round_down(old_total + diff, fs_info->sectorsize));
2836         device->fs_devices->total_rw_bytes += diff;
2837
2838         btrfs_device_set_total_bytes(device, new_size);
2839         btrfs_device_set_disk_total_bytes(device, new_size);
2840         btrfs_clear_space_info_full(device->fs_info);
2841         if (list_empty(&device->post_commit_list))
2842                 list_add_tail(&device->post_commit_list,
2843                               &trans->transaction->dev_update_list);
2844         mutex_unlock(&fs_info->chunk_mutex);
2845
2846         return btrfs_update_device(trans, device);
2847 }
2848
2849 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2850 {
2851         struct btrfs_fs_info *fs_info = trans->fs_info;
2852         struct btrfs_root *root = fs_info->chunk_root;
2853         int ret;
2854         struct btrfs_path *path;
2855         struct btrfs_key key;
2856
2857         path = btrfs_alloc_path();
2858         if (!path)
2859                 return -ENOMEM;
2860
2861         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2862         key.offset = chunk_offset;
2863         key.type = BTRFS_CHUNK_ITEM_KEY;
2864
2865         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2866         if (ret < 0)
2867                 goto out;
2868         else if (ret > 0) { /* Logic error or corruption */
2869                 btrfs_handle_fs_error(fs_info, -ENOENT,
2870                                       "Failed lookup while freeing chunk.");
2871                 ret = -ENOENT;
2872                 goto out;
2873         }
2874
2875         ret = btrfs_del_item(trans, root, path);
2876         if (ret < 0)
2877                 btrfs_handle_fs_error(fs_info, ret,
2878                                       "Failed to delete chunk item.");
2879 out:
2880         btrfs_free_path(path);
2881         return ret;
2882 }
2883
2884 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2885 {
2886         struct btrfs_super_block *super_copy = fs_info->super_copy;
2887         struct btrfs_disk_key *disk_key;
2888         struct btrfs_chunk *chunk;
2889         u8 *ptr;
2890         int ret = 0;
2891         u32 num_stripes;
2892         u32 array_size;
2893         u32 len = 0;
2894         u32 cur;
2895         struct btrfs_key key;
2896
2897         mutex_lock(&fs_info->chunk_mutex);
2898         array_size = btrfs_super_sys_array_size(super_copy);
2899
2900         ptr = super_copy->sys_chunk_array;
2901         cur = 0;
2902
2903         while (cur < array_size) {
2904                 disk_key = (struct btrfs_disk_key *)ptr;
2905                 btrfs_disk_key_to_cpu(&key, disk_key);
2906
2907                 len = sizeof(*disk_key);
2908
2909                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2910                         chunk = (struct btrfs_chunk *)(ptr + len);
2911                         num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2912                         len += btrfs_chunk_item_size(num_stripes);
2913                 } else {
2914                         ret = -EIO;
2915                         break;
2916                 }
2917                 if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2918                     key.offset == chunk_offset) {
2919                         memmove(ptr, ptr + len, array_size - (cur + len));
2920                         array_size -= len;
2921                         btrfs_set_super_sys_array_size(super_copy, array_size);
2922                 } else {
2923                         ptr += len;
2924                         cur += len;
2925                 }
2926         }
2927         mutex_unlock(&fs_info->chunk_mutex);
2928         return ret;
2929 }
2930
2931 /*
2932  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2933  * @logical: Logical block offset in bytes.
2934  * @length: Length of extent in bytes.
2935  *
2936  * Return: Chunk mapping or ERR_PTR.
2937  */
2938 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
2939                                        u64 logical, u64 length)
2940 {
2941         struct extent_map_tree *em_tree;
2942         struct extent_map *em;
2943
2944         em_tree = &fs_info->mapping_tree.map_tree;
2945         read_lock(&em_tree->lock);
2946         em = lookup_extent_mapping(em_tree, logical, length);
2947         read_unlock(&em_tree->lock);
2948
2949         if (!em) {
2950                 btrfs_crit(fs_info, "unable to find logical %llu length %llu",
2951                            logical, length);
2952                 return ERR_PTR(-EINVAL);
2953         }
2954
2955         if (em->start > logical || em->start + em->len < logical) {
2956                 btrfs_crit(fs_info,
2957                            "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
2958                            logical, length, em->start, em->start + em->len);
2959                 free_extent_map(em);
2960                 return ERR_PTR(-EINVAL);
2961         }
2962
2963         /* callers are responsible for dropping em's ref. */
2964         return em;
2965 }
2966
2967 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2968 {
2969         struct btrfs_fs_info *fs_info = trans->fs_info;
2970         struct extent_map *em;
2971         struct map_lookup *map;
2972         u64 dev_extent_len = 0;
2973         int i, ret = 0;
2974         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2975
2976         em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
2977         if (IS_ERR(em)) {
2978                 /*
2979                  * This is a logic error, but we don't want to just rely on the
2980                  * user having built with ASSERT enabled, so if ASSERT doesn't
2981                  * do anything we still error out.
2982                  */
2983                 ASSERT(0);
2984                 return PTR_ERR(em);
2985         }
2986         map = em->map_lookup;
2987         mutex_lock(&fs_info->chunk_mutex);
2988         check_system_chunk(trans, map->type);
2989         mutex_unlock(&fs_info->chunk_mutex);
2990
2991         /*
2992          * Take the device list mutex to prevent races with the final phase of
2993          * a device replace operation that replaces the device object associated
2994          * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
2995          */
2996         mutex_lock(&fs_devices->device_list_mutex);
2997         for (i = 0; i < map->num_stripes; i++) {
2998                 struct btrfs_device *device = map->stripes[i].dev;
2999                 ret = btrfs_free_dev_extent(trans, device,
3000                                             map->stripes[i].physical,
3001                                             &dev_extent_len);
3002                 if (ret) {
3003                         mutex_unlock(&fs_devices->device_list_mutex);
3004                         btrfs_abort_transaction(trans, ret);
3005                         goto out;
3006                 }
3007
3008                 if (device->bytes_used > 0) {
3009                         mutex_lock(&fs_info->chunk_mutex);
3010                         btrfs_device_set_bytes_used(device,
3011                                         device->bytes_used - dev_extent_len);
3012                         atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
3013                         btrfs_clear_space_info_full(fs_info);
3014                         mutex_unlock(&fs_info->chunk_mutex);
3015                 }
3016
3017                 ret = btrfs_update_device(trans, device);
3018                 if (ret) {
3019                         mutex_unlock(&fs_devices->device_list_mutex);
3020                         btrfs_abort_transaction(trans, ret);
3021                         goto out;
3022                 }
3023         }
3024         mutex_unlock(&fs_devices->device_list_mutex);
3025
3026         ret = btrfs_free_chunk(trans, chunk_offset);
3027         if (ret) {
3028                 btrfs_abort_transaction(trans, ret);
3029                 goto out;
3030         }
3031
3032         trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
3033
3034         if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3035                 ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3036                 if (ret) {
3037                         btrfs_abort_transaction(trans, ret);
3038                         goto out;
3039                 }
3040         }
3041
3042         ret = btrfs_remove_block_group(trans, chunk_offset, em);
3043         if (ret) {
3044                 btrfs_abort_transaction(trans, ret);
3045                 goto out;
3046         }
3047
3048 out:
3049         /* once for us */
3050         free_extent_map(em);
3051         return ret;
3052 }
3053
3054 static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3055 {
3056         struct btrfs_root *root = fs_info->chunk_root;
3057         struct btrfs_trans_handle *trans;
3058         int ret;
3059
3060         /*
3061          * Prevent races with automatic removal of unused block groups.
3062          * After we relocate and before we remove the chunk with offset
3063          * chunk_offset, automatic removal of the block group can kick in,
3064          * resulting in a failure when calling btrfs_remove_chunk() below.
3065          *
3066          * Make sure to acquire this mutex before doing a tree search (dev
3067          * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3068          * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3069          * we release the path used to search the chunk/dev tree and before
3070          * the current task acquires this mutex and calls us.
3071          */
3072         lockdep_assert_held(&fs_info->delete_unused_bgs_mutex);
3073
3074         ret = btrfs_can_relocate(fs_info, chunk_offset);
3075         if (ret)
3076                 return -ENOSPC;
3077
3078         /* step one, relocate all the extents inside this chunk */
3079         btrfs_scrub_pause(fs_info);
3080         ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3081         btrfs_scrub_continue(fs_info);
3082         if (ret)
3083                 return ret;
3084
3085         /*
3086          * We add the kobjects here (and after forcing data chunk creation)
3087          * since relocation is the only place we'll create chunks of a new
3088          * type at runtime.  The only place where we'll remove the last
3089          * chunk of a type is the call immediately below this one.  Even
3090          * so, we're protected against races with the cleaner thread since
3091          * we're covered by the delete_unused_bgs_mutex.
3092          */
3093         btrfs_add_raid_kobjects(fs_info);
3094
3095         trans = btrfs_start_trans_remove_block_group(root->fs_info,
3096                                                      chunk_offset);
3097         if (IS_ERR(trans)) {
3098                 ret = PTR_ERR(trans);
3099                 btrfs_handle_fs_error(root->fs_info, ret, NULL);
3100                 return ret;
3101         }
3102
3103         /*
3104          * step two, delete the device extents and the
3105          * chunk tree entries
3106          */
3107         ret = btrfs_remove_chunk(trans, chunk_offset);
3108         btrfs_end_transaction(trans);
3109         return ret;
3110 }
3111
3112 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3113 {
3114         struct btrfs_root *chunk_root = fs_info->chunk_root;
3115         struct btrfs_path *path;
3116         struct extent_buffer *leaf;
3117         struct btrfs_chunk *chunk;
3118         struct btrfs_key key;
3119         struct btrfs_key found_key;
3120         u64 chunk_type;
3121         bool retried = false;
3122         int failed = 0;
3123         int ret;
3124
3125         path = btrfs_alloc_path();
3126         if (!path)
3127                 return -ENOMEM;
3128
3129 again:
3130         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3131         key.offset = (u64)-1;
3132         key.type = BTRFS_CHUNK_ITEM_KEY;
3133
3134         while (1) {
3135                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
3136                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3137                 if (ret < 0) {
3138                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3139                         goto error;
3140                 }
3141                 BUG_ON(ret == 0); /* Corruption */
3142
3143                 ret = btrfs_previous_item(chunk_root, path, key.objectid,
3144                                           key.type);
3145                 if (ret)
3146                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3147                 if (ret < 0)
3148                         goto error;
3149                 if (ret > 0)
3150                         break;
3151
3152                 leaf = path->nodes[0];
3153                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3154
3155                 chunk = btrfs_item_ptr(leaf, path->slots[0],
3156                                        struct btrfs_chunk);
3157                 chunk_type = btrfs_chunk_type(leaf, chunk);
3158                 btrfs_release_path(path);
3159
3160                 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3161                         ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3162                         if (ret == -ENOSPC)
3163                                 failed++;
3164                         else
3165                                 BUG_ON(ret);
3166                 }
3167                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3168
3169                 if (found_key.offset == 0)
3170                         break;
3171                 key.offset = found_key.offset - 1;
3172         }
3173         ret = 0;
3174         if (failed && !retried) {
3175                 failed = 0;
3176                 retried = true;
3177                 goto again;
3178         } else if (WARN_ON(failed && retried)) {
3179                 ret = -ENOSPC;
3180         }
3181 error:
3182         btrfs_free_path(path);
3183         return ret;
3184 }
3185
3186 /*
3187  * return 1 : allocate a data chunk successfully,
3188  * return <0: errors during allocating a data chunk,
3189  * return 0 : no need to allocate a data chunk.
3190  */
3191 static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3192                                       u64 chunk_offset)
3193 {
3194         struct btrfs_block_group_cache *cache;
3195         u64 bytes_used;
3196         u64 chunk_type;
3197
3198         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3199         ASSERT(cache);
3200         chunk_type = cache->flags;
3201         btrfs_put_block_group(cache);
3202
3203         if (chunk_type & BTRFS_BLOCK_GROUP_DATA) {
3204                 spin_lock(&fs_info->data_sinfo->lock);
3205                 bytes_used = fs_info->data_sinfo->bytes_used;
3206                 spin_unlock(&fs_info->data_sinfo->lock);
3207
3208                 if (!bytes_used) {
3209                         struct btrfs_trans_handle *trans;
3210                         int ret;
3211
3212                         trans = btrfs_join_transaction(fs_info->tree_root);
3213                         if (IS_ERR(trans))
3214                                 return PTR_ERR(trans);
3215
3216                         ret = btrfs_force_chunk_alloc(trans,
3217                                                       BTRFS_BLOCK_GROUP_DATA);
3218                         btrfs_end_transaction(trans);
3219          &n