btrfs: get fs_info from eb in should_balance_chunk
[sfrench/cifs-2.6.git] / fs / btrfs / volumes.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/ratelimit.h>
12 #include <linux/kthread.h>
13 #include <linux/raid/pq.h>
14 #include <linux/semaphore.h>
15 #include <linux/uuid.h>
16 #include <linux/list_sort.h>
17 #include "ctree.h"
18 #include "extent_map.h"
19 #include "disk-io.h"
20 #include "transaction.h"
21 #include "print-tree.h"
22 #include "volumes.h"
23 #include "raid56.h"
24 #include "async-thread.h"
25 #include "check-integrity.h"
26 #include "rcu-string.h"
27 #include "math.h"
28 #include "dev-replace.h"
29 #include "sysfs.h"
30 #include "tree-checker.h"
31
32 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
33         [BTRFS_RAID_RAID10] = {
34                 .sub_stripes    = 2,
35                 .dev_stripes    = 1,
36                 .devs_max       = 0,    /* 0 == as many as possible */
37                 .devs_min       = 4,
38                 .tolerated_failures = 1,
39                 .devs_increment = 2,
40                 .ncopies        = 2,
41                 .nparity        = 0,
42                 .raid_name      = "raid10",
43                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID10,
44                 .mindev_error   = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
45         },
46         [BTRFS_RAID_RAID1] = {
47                 .sub_stripes    = 1,
48                 .dev_stripes    = 1,
49                 .devs_max       = 2,
50                 .devs_min       = 2,
51                 .tolerated_failures = 1,
52                 .devs_increment = 2,
53                 .ncopies        = 2,
54                 .nparity        = 0,
55                 .raid_name      = "raid1",
56                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1,
57                 .mindev_error   = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
58         },
59         [BTRFS_RAID_DUP] = {
60                 .sub_stripes    = 1,
61                 .dev_stripes    = 2,
62                 .devs_max       = 1,
63                 .devs_min       = 1,
64                 .tolerated_failures = 0,
65                 .devs_increment = 1,
66                 .ncopies        = 2,
67                 .nparity        = 0,
68                 .raid_name      = "dup",
69                 .bg_flag        = BTRFS_BLOCK_GROUP_DUP,
70                 .mindev_error   = 0,
71         },
72         [BTRFS_RAID_RAID0] = {
73                 .sub_stripes    = 1,
74                 .dev_stripes    = 1,
75                 .devs_max       = 0,
76                 .devs_min       = 2,
77                 .tolerated_failures = 0,
78                 .devs_increment = 1,
79                 .ncopies        = 1,
80                 .nparity        = 0,
81                 .raid_name      = "raid0",
82                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID0,
83                 .mindev_error   = 0,
84         },
85         [BTRFS_RAID_SINGLE] = {
86                 .sub_stripes    = 1,
87                 .dev_stripes    = 1,
88                 .devs_max       = 1,
89                 .devs_min       = 1,
90                 .tolerated_failures = 0,
91                 .devs_increment = 1,
92                 .ncopies        = 1,
93                 .nparity        = 0,
94                 .raid_name      = "single",
95                 .bg_flag        = 0,
96                 .mindev_error   = 0,
97         },
98         [BTRFS_RAID_RAID5] = {
99                 .sub_stripes    = 1,
100                 .dev_stripes    = 1,
101                 .devs_max       = 0,
102                 .devs_min       = 2,
103                 .tolerated_failures = 1,
104                 .devs_increment = 1,
105                 .ncopies        = 1,
106                 .nparity        = 1,
107                 .raid_name      = "raid5",
108                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID5,
109                 .mindev_error   = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
110         },
111         [BTRFS_RAID_RAID6] = {
112                 .sub_stripes    = 1,
113                 .dev_stripes    = 1,
114                 .devs_max       = 0,
115                 .devs_min       = 3,
116                 .tolerated_failures = 2,
117                 .devs_increment = 1,
118                 .ncopies        = 1,
119                 .nparity        = 2,
120                 .raid_name      = "raid6",
121                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID6,
122                 .mindev_error   = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
123         },
124 };
125
126 const char *get_raid_name(enum btrfs_raid_types type)
127 {
128         if (type >= BTRFS_NR_RAID_TYPES)
129                 return NULL;
130
131         return btrfs_raid_array[type].raid_name;
132 }
133
134 /*
135  * Fill @buf with textual description of @bg_flags, no more than @size_buf
136  * bytes including terminating null byte.
137  */
138 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
139 {
140         int i;
141         int ret;
142         char *bp = buf;
143         u64 flags = bg_flags;
144         u32 size_bp = size_buf;
145
146         if (!flags) {
147                 strcpy(bp, "NONE");
148                 return;
149         }
150
151 #define DESCRIBE_FLAG(flag, desc)                                               \
152         do {                                                            \
153                 if (flags & (flag)) {                                   \
154                         ret = snprintf(bp, size_bp, "%s|", (desc));     \
155                         if (ret < 0 || ret >= size_bp)                  \
156                                 goto out_overflow;                      \
157                         size_bp -= ret;                                 \
158                         bp += ret;                                      \
159                         flags &= ~(flag);                               \
160                 }                                                       \
161         } while (0)
162
163         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
164         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
165         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
166
167         DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
168         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
169                 DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
170                               btrfs_raid_array[i].raid_name);
171 #undef DESCRIBE_FLAG
172
173         if (flags) {
174                 ret = snprintf(bp, size_bp, "0x%llx|", flags);
175                 size_bp -= ret;
176         }
177
178         if (size_bp < size_buf)
179                 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
180
181         /*
182          * The text is trimmed, it's up to the caller to provide sufficiently
183          * large buffer
184          */
185 out_overflow:;
186 }
187
188 static int init_first_rw_device(struct btrfs_trans_handle *trans,
189                                 struct btrfs_fs_info *fs_info);
190 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
191 static void __btrfs_reset_dev_stats(struct btrfs_device *dev);
192 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
193 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
194 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
195                              enum btrfs_map_op op,
196                              u64 logical, u64 *length,
197                              struct btrfs_bio **bbio_ret,
198                              int mirror_num, int need_raid_map);
199
200 /*
201  * Device locking
202  * ==============
203  *
204  * There are several mutexes that protect manipulation of devices and low-level
205  * structures like chunks but not block groups, extents or files
206  *
207  * uuid_mutex (global lock)
208  * ------------------------
209  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
210  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
211  * device) or requested by the device= mount option
212  *
213  * the mutex can be very coarse and can cover long-running operations
214  *
215  * protects: updates to fs_devices counters like missing devices, rw devices,
216  * seeding, structure cloning, opening/closing devices at mount/umount time
217  *
218  * global::fs_devs - add, remove, updates to the global list
219  *
220  * does not protect: manipulation of the fs_devices::devices list!
221  *
222  * btrfs_device::name - renames (write side), read is RCU
223  *
224  * fs_devices::device_list_mutex (per-fs, with RCU)
225  * ------------------------------------------------
226  * protects updates to fs_devices::devices, ie. adding and deleting
227  *
228  * simple list traversal with read-only actions can be done with RCU protection
229  *
230  * may be used to exclude some operations from running concurrently without any
231  * modifications to the list (see write_all_supers)
232  *
233  * balance_mutex
234  * -------------
235  * protects balance structures (status, state) and context accessed from
236  * several places (internally, ioctl)
237  *
238  * chunk_mutex
239  * -----------
240  * protects chunks, adding or removing during allocation, trim or when a new
241  * device is added/removed
242  *
243  * cleaner_mutex
244  * -------------
245  * a big lock that is held by the cleaner thread and prevents running subvolume
246  * cleaning together with relocation or delayed iputs
247  *
248  *
249  * Lock nesting
250  * ============
251  *
252  * uuid_mutex
253  *   volume_mutex
254  *     device_list_mutex
255  *       chunk_mutex
256  *     balance_mutex
257  *
258  *
259  * Exclusive operations, BTRFS_FS_EXCL_OP
260  * ======================================
261  *
262  * Maintains the exclusivity of the following operations that apply to the
263  * whole filesystem and cannot run in parallel.
264  *
265  * - Balance (*)
266  * - Device add
267  * - Device remove
268  * - Device replace (*)
269  * - Resize
270  *
271  * The device operations (as above) can be in one of the following states:
272  *
273  * - Running state
274  * - Paused state
275  * - Completed state
276  *
277  * Only device operations marked with (*) can go into the Paused state for the
278  * following reasons:
279  *
280  * - ioctl (only Balance can be Paused through ioctl)
281  * - filesystem remounted as read-only
282  * - filesystem unmounted and mounted as read-only
283  * - system power-cycle and filesystem mounted as read-only
284  * - filesystem or device errors leading to forced read-only
285  *
286  * BTRFS_FS_EXCL_OP flag is set and cleared using atomic operations.
287  * During the course of Paused state, the BTRFS_FS_EXCL_OP remains set.
288  * A device operation in Paused or Running state can be canceled or resumed
289  * either by ioctl (Balance only) or when remounted as read-write.
290  * BTRFS_FS_EXCL_OP flag is cleared when the device operation is canceled or
291  * completed.
292  */
293
294 DEFINE_MUTEX(uuid_mutex);
295 static LIST_HEAD(fs_uuids);
296 struct list_head *btrfs_get_fs_uuids(void)
297 {
298         return &fs_uuids;
299 }
300
301 /*
302  * alloc_fs_devices - allocate struct btrfs_fs_devices
303  * @fsid:               if not NULL, copy the UUID to fs_devices::fsid
304  * @metadata_fsid:      if not NULL, copy the UUID to fs_devices::metadata_fsid
305  *
306  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
307  * The returned struct is not linked onto any lists and can be destroyed with
308  * kfree() right away.
309  */
310 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
311                                                  const u8 *metadata_fsid)
312 {
313         struct btrfs_fs_devices *fs_devs;
314
315         fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
316         if (!fs_devs)
317                 return ERR_PTR(-ENOMEM);
318
319         mutex_init(&fs_devs->device_list_mutex);
320
321         INIT_LIST_HEAD(&fs_devs->devices);
322         INIT_LIST_HEAD(&fs_devs->alloc_list);
323         INIT_LIST_HEAD(&fs_devs->fs_list);
324         if (fsid)
325                 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
326
327         if (metadata_fsid)
328                 memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
329         else if (fsid)
330                 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
331
332         return fs_devs;
333 }
334
335 void btrfs_free_device(struct btrfs_device *device)
336 {
337         WARN_ON(!list_empty(&device->post_commit_list));
338         rcu_string_free(device->name);
339         extent_io_tree_release(&device->alloc_state);
340         bio_put(device->flush_bio);
341         kfree(device);
342 }
343
344 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
345 {
346         struct btrfs_device *device;
347         WARN_ON(fs_devices->opened);
348         while (!list_empty(&fs_devices->devices)) {
349                 device = list_entry(fs_devices->devices.next,
350                                     struct btrfs_device, dev_list);
351                 list_del(&device->dev_list);
352                 btrfs_free_device(device);
353         }
354         kfree(fs_devices);
355 }
356
357 static void btrfs_kobject_uevent(struct block_device *bdev,
358                                  enum kobject_action action)
359 {
360         int ret;
361
362         ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
363         if (ret)
364                 pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
365                         action,
366                         kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
367                         &disk_to_dev(bdev->bd_disk)->kobj);
368 }
369
370 void __exit btrfs_cleanup_fs_uuids(void)
371 {
372         struct btrfs_fs_devices *fs_devices;
373
374         while (!list_empty(&fs_uuids)) {
375                 fs_devices = list_entry(fs_uuids.next,
376                                         struct btrfs_fs_devices, fs_list);
377                 list_del(&fs_devices->fs_list);
378                 free_fs_devices(fs_devices);
379         }
380 }
381
382 /*
383  * Returns a pointer to a new btrfs_device on success; ERR_PTR() on error.
384  * Returned struct is not linked onto any lists and must be destroyed using
385  * btrfs_free_device.
386  */
387 static struct btrfs_device *__alloc_device(void)
388 {
389         struct btrfs_device *dev;
390
391         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
392         if (!dev)
393                 return ERR_PTR(-ENOMEM);
394
395         /*
396          * Preallocate a bio that's always going to be used for flushing device
397          * barriers and matches the device lifespan
398          */
399         dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
400         if (!dev->flush_bio) {
401                 kfree(dev);
402                 return ERR_PTR(-ENOMEM);
403         }
404
405         INIT_LIST_HEAD(&dev->dev_list);
406         INIT_LIST_HEAD(&dev->dev_alloc_list);
407         INIT_LIST_HEAD(&dev->post_commit_list);
408
409         spin_lock_init(&dev->io_lock);
410
411         atomic_set(&dev->reada_in_flight, 0);
412         atomic_set(&dev->dev_stats_ccnt, 0);
413         btrfs_device_data_ordered_init(dev);
414         INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
415         INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
416         extent_io_tree_init(NULL, &dev->alloc_state, 0, NULL);
417
418         return dev;
419 }
420
421 static noinline struct btrfs_fs_devices *find_fsid(
422                 const u8 *fsid, const u8 *metadata_fsid)
423 {
424         struct btrfs_fs_devices *fs_devices;
425
426         ASSERT(fsid);
427
428         if (metadata_fsid) {
429                 /*
430                  * Handle scanned device having completed its fsid change but
431                  * belonging to a fs_devices that was created by first scanning
432                  * a device which didn't have its fsid/metadata_uuid changed
433                  * at all and the CHANGING_FSID_V2 flag set.
434                  */
435                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
436                         if (fs_devices->fsid_change &&
437                             memcmp(metadata_fsid, fs_devices->fsid,
438                                    BTRFS_FSID_SIZE) == 0 &&
439                             memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
440                                    BTRFS_FSID_SIZE) == 0) {
441                                 return fs_devices;
442                         }
443                 }
444                 /*
445                  * Handle scanned device having completed its fsid change but
446                  * belonging to a fs_devices that was created by a device that
447                  * has an outdated pair of fsid/metadata_uuid and
448                  * CHANGING_FSID_V2 flag set.
449                  */
450                 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
451                         if (fs_devices->fsid_change &&
452                             memcmp(fs_devices->metadata_uuid,
453                                    fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
454                             memcmp(metadata_fsid, fs_devices->metadata_uuid,
455                                    BTRFS_FSID_SIZE) == 0) {
456                                 return fs_devices;
457                         }
458                 }
459         }
460
461         /* Handle non-split brain cases */
462         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
463                 if (metadata_fsid) {
464                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
465                             && memcmp(metadata_fsid, fs_devices->metadata_uuid,
466                                       BTRFS_FSID_SIZE) == 0)
467                                 return fs_devices;
468                 } else {
469                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
470                                 return fs_devices;
471                 }
472         }
473         return NULL;
474 }
475
476 static int
477 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
478                       int flush, struct block_device **bdev,
479                       struct buffer_head **bh)
480 {
481         int ret;
482
483         *bdev = blkdev_get_by_path(device_path, flags, holder);
484
485         if (IS_ERR(*bdev)) {
486                 ret = PTR_ERR(*bdev);
487                 goto error;
488         }
489
490         if (flush)
491                 filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
492         ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
493         if (ret) {
494                 blkdev_put(*bdev, flags);
495                 goto error;
496         }
497         invalidate_bdev(*bdev);
498         *bh = btrfs_read_dev_super(*bdev);
499         if (IS_ERR(*bh)) {
500                 ret = PTR_ERR(*bh);
501                 blkdev_put(*bdev, flags);
502                 goto error;
503         }
504
505         return 0;
506
507 error:
508         *bdev = NULL;
509         *bh = NULL;
510         return ret;
511 }
512
513 static void requeue_list(struct btrfs_pending_bios *pending_bios,
514                         struct bio *head, struct bio *tail)
515 {
516
517         struct bio *old_head;
518
519         old_head = pending_bios->head;
520         pending_bios->head = head;
521         if (pending_bios->tail)
522                 tail->bi_next = old_head;
523         else
524                 pending_bios->tail = tail;
525 }
526
527 /*
528  * we try to collect pending bios for a device so we don't get a large
529  * number of procs sending bios down to the same device.  This greatly
530  * improves the schedulers ability to collect and merge the bios.
531  *
532  * But, it also turns into a long list of bios to process and that is sure
533  * to eventually make the worker thread block.  The solution here is to
534  * make some progress and then put this work struct back at the end of
535  * the list if the block device is congested.  This way, multiple devices
536  * can make progress from a single worker thread.
537  */
538 static noinline void run_scheduled_bios(struct btrfs_device *device)
539 {
540         struct btrfs_fs_info *fs_info = device->fs_info;
541         struct bio *pending;
542         struct backing_dev_info *bdi;
543         struct btrfs_pending_bios *pending_bios;
544         struct bio *tail;
545         struct bio *cur;
546         int again = 0;
547         unsigned long num_run;
548         unsigned long batch_run = 0;
549         unsigned long last_waited = 0;
550         int force_reg = 0;
551         int sync_pending = 0;
552         struct blk_plug plug;
553
554         /*
555          * this function runs all the bios we've collected for
556          * a particular device.  We don't want to wander off to
557          * another device without first sending all of these down.
558          * So, setup a plug here and finish it off before we return
559          */
560         blk_start_plug(&plug);
561
562         bdi = device->bdev->bd_bdi;
563
564 loop:
565         spin_lock(&device->io_lock);
566
567 loop_lock:
568         num_run = 0;
569
570         /* take all the bios off the list at once and process them
571          * later on (without the lock held).  But, remember the
572          * tail and other pointers so the bios can be properly reinserted
573          * into the list if we hit congestion
574          */
575         if (!force_reg && device->pending_sync_bios.head) {
576                 pending_bios = &device->pending_sync_bios;
577                 force_reg = 1;
578         } else {
579                 pending_bios = &device->pending_bios;
580                 force_reg = 0;
581         }
582
583         pending = pending_bios->head;
584         tail = pending_bios->tail;
585         WARN_ON(pending && !tail);
586
587         /*
588          * if pending was null this time around, no bios need processing
589          * at all and we can stop.  Otherwise it'll loop back up again
590          * and do an additional check so no bios are missed.
591          *
592          * device->running_pending is used to synchronize with the
593          * schedule_bio code.
594          */
595         if (device->pending_sync_bios.head == NULL &&
596             device->pending_bios.head == NULL) {
597                 again = 0;
598                 device->running_pending = 0;
599         } else {
600                 again = 1;
601                 device->running_pending = 1;
602         }
603
604         pending_bios->head = NULL;
605         pending_bios->tail = NULL;
606
607         spin_unlock(&device->io_lock);
608
609         while (pending) {
610
611                 rmb();
612                 /* we want to work on both lists, but do more bios on the
613                  * sync list than the regular list
614                  */
615                 if ((num_run > 32 &&
616                     pending_bios != &device->pending_sync_bios &&
617                     device->pending_sync_bios.head) ||
618                    (num_run > 64 && pending_bios == &device->pending_sync_bios &&
619                     device->pending_bios.head)) {
620                         spin_lock(&device->io_lock);
621                         requeue_list(pending_bios, pending, tail);
622                         goto loop_lock;
623                 }
624
625                 cur = pending;
626                 pending = pending->bi_next;
627                 cur->bi_next = NULL;
628
629                 BUG_ON(atomic_read(&cur->__bi_cnt) == 0);
630
631                 /*
632                  * if we're doing the sync list, record that our
633                  * plug has some sync requests on it
634                  *
635                  * If we're doing the regular list and there are
636                  * sync requests sitting around, unplug before
637                  * we add more
638                  */
639                 if (pending_bios == &device->pending_sync_bios) {
640                         sync_pending = 1;
641                 } else if (sync_pending) {
642                         blk_finish_plug(&plug);
643                         blk_start_plug(&plug);
644                         sync_pending = 0;
645                 }
646
647                 btrfsic_submit_bio(cur);
648                 num_run++;
649                 batch_run++;
650
651                 cond_resched();
652
653                 /*
654                  * we made progress, there is more work to do and the bdi
655                  * is now congested.  Back off and let other work structs
656                  * run instead
657                  */
658                 if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
659                     fs_info->fs_devices->open_devices > 1) {
660                         struct io_context *ioc;
661
662                         ioc = current->io_context;
663
664                         /*
665                          * the main goal here is that we don't want to
666                          * block if we're going to be able to submit
667                          * more requests without blocking.
668                          *
669                          * This code does two great things, it pokes into
670                          * the elevator code from a filesystem _and_
671                          * it makes assumptions about how batching works.
672                          */
673                         if (ioc && ioc->nr_batch_requests > 0 &&
674                             time_before(jiffies, ioc->last_waited + HZ/50UL) &&
675                             (last_waited == 0 ||
676                              ioc->last_waited == last_waited)) {
677                                 /*
678                                  * we want to go through our batch of
679                                  * requests and stop.  So, we copy out
680                                  * the ioc->last_waited time and test
681                                  * against it before looping
682                                  */
683                                 last_waited = ioc->last_waited;
684                                 cond_resched();
685                                 continue;
686                         }
687                         spin_lock(&device->io_lock);
688                         requeue_list(pending_bios, pending, tail);
689                         device->running_pending = 1;
690
691                         spin_unlock(&device->io_lock);
692                         btrfs_queue_work(fs_info->submit_workers,
693                                          &device->work);
694                         goto done;
695                 }
696         }
697
698         cond_resched();
699         if (again)
700                 goto loop;
701
702         spin_lock(&device->io_lock);
703         if (device->pending_bios.head || device->pending_sync_bios.head)
704                 goto loop_lock;
705         spin_unlock(&device->io_lock);
706
707 done:
708         blk_finish_plug(&plug);
709 }
710
711 static void pending_bios_fn(struct btrfs_work *work)
712 {
713         struct btrfs_device *device;
714
715         device = container_of(work, struct btrfs_device, work);
716         run_scheduled_bios(device);
717 }
718
719 static bool device_path_matched(const char *path, struct btrfs_device *device)
720 {
721         int found;
722
723         rcu_read_lock();
724         found = strcmp(rcu_str_deref(device->name), path);
725         rcu_read_unlock();
726
727         return found == 0;
728 }
729
730 /*
731  *  Search and remove all stale (devices which are not mounted) devices.
732  *  When both inputs are NULL, it will search and release all stale devices.
733  *  path:       Optional. When provided will it release all unmounted devices
734  *              matching this path only.
735  *  skip_dev:   Optional. Will skip this device when searching for the stale
736  *              devices.
737  *  Return:     0 for success or if @path is NULL.
738  *              -EBUSY if @path is a mounted device.
739  *              -ENOENT if @path does not match any device in the list.
740  */
741 static int btrfs_free_stale_devices(const char *path,
742                                      struct btrfs_device *skip_device)
743 {
744         struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
745         struct btrfs_device *device, *tmp_device;
746         int ret = 0;
747
748         if (path)
749                 ret = -ENOENT;
750
751         list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
752
753                 mutex_lock(&fs_devices->device_list_mutex);
754                 list_for_each_entry_safe(device, tmp_device,
755                                          &fs_devices->devices, dev_list) {
756                         if (skip_device && skip_device == device)
757                                 continue;
758                         if (path && !device->name)
759                                 continue;
760                         if (path && !device_path_matched(path, device))
761                                 continue;
762                         if (fs_devices->opened) {
763                                 /* for an already deleted device return 0 */
764                                 if (path && ret != 0)
765                                         ret = -EBUSY;
766                                 break;
767                         }
768
769                         /* delete the stale device */
770                         fs_devices->num_devices--;
771                         list_del(&device->dev_list);
772                         btrfs_free_device(device);
773
774                         ret = 0;
775                         if (fs_devices->num_devices == 0)
776                                 break;
777                 }
778                 mutex_unlock(&fs_devices->device_list_mutex);
779
780                 if (fs_devices->num_devices == 0) {
781                         btrfs_sysfs_remove_fsid(fs_devices);
782                         list_del(&fs_devices->fs_list);
783                         free_fs_devices(fs_devices);
784                 }
785         }
786
787         return ret;
788 }
789
790 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
791                         struct btrfs_device *device, fmode_t flags,
792                         void *holder)
793 {
794         struct request_queue *q;
795         struct block_device *bdev;
796         struct buffer_head *bh;
797         struct btrfs_super_block *disk_super;
798         u64 devid;
799         int ret;
800
801         if (device->bdev)
802                 return -EINVAL;
803         if (!device->name)
804                 return -EINVAL;
805
806         ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
807                                     &bdev, &bh);
808         if (ret)
809                 return ret;
810
811         disk_super = (struct btrfs_super_block *)bh->b_data;
812         devid = btrfs_stack_device_id(&disk_super->dev_item);
813         if (devid != device->devid)
814                 goto error_brelse;
815
816         if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
817                 goto error_brelse;
818
819         device->generation = btrfs_super_generation(disk_super);
820
821         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
822                 if (btrfs_super_incompat_flags(disk_super) &
823                     BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
824                         pr_err(
825                 "BTRFS: Invalid seeding and uuid-changed device detected\n");
826                         goto error_brelse;
827                 }
828
829                 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
830                 fs_devices->seeding = 1;
831         } else {
832                 if (bdev_read_only(bdev))
833                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
834                 else
835                         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
836         }
837
838         q = bdev_get_queue(bdev);
839         if (!blk_queue_nonrot(q))
840                 fs_devices->rotating = 1;
841
842         device->bdev = bdev;
843         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
844         device->mode = flags;
845
846         fs_devices->open_devices++;
847         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
848             device->devid != BTRFS_DEV_REPLACE_DEVID) {
849                 fs_devices->rw_devices++;
850                 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
851         }
852         brelse(bh);
853
854         return 0;
855
856 error_brelse:
857         brelse(bh);
858         blkdev_put(bdev, flags);
859
860         return -EINVAL;
861 }
862
863 /*
864  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
865  * being created with a disk that has already completed its fsid change.
866  */
867 static struct btrfs_fs_devices *find_fsid_inprogress(
868                                         struct btrfs_super_block *disk_super)
869 {
870         struct btrfs_fs_devices *fs_devices;
871
872         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
873                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
874                            BTRFS_FSID_SIZE) != 0 &&
875                     memcmp(fs_devices->metadata_uuid, disk_super->fsid,
876                            BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
877                         return fs_devices;
878                 }
879         }
880
881         return NULL;
882 }
883
884
885 static struct btrfs_fs_devices *find_fsid_changed(
886                                         struct btrfs_super_block *disk_super)
887 {
888         struct btrfs_fs_devices *fs_devices;
889
890         /*
891          * Handles the case where scanned device is part of an fs that had
892          * multiple successful changes of FSID but curently device didn't
893          * observe it. Meaning our fsid will be different than theirs.
894          */
895         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
896                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
897                            BTRFS_FSID_SIZE) != 0 &&
898                     memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
899                            BTRFS_FSID_SIZE) == 0 &&
900                     memcmp(fs_devices->fsid, disk_super->fsid,
901                            BTRFS_FSID_SIZE) != 0) {
902                         return fs_devices;
903                 }
904         }
905
906         return NULL;
907 }
908 /*
909  * Add new device to list of registered devices
910  *
911  * Returns:
912  * device pointer which was just added or updated when successful
913  * error pointer when failed
914  */
915 static noinline struct btrfs_device *device_list_add(const char *path,
916                            struct btrfs_super_block *disk_super,
917                            bool *new_device_added)
918 {
919         struct btrfs_device *device;
920         struct btrfs_fs_devices *fs_devices = NULL;
921         struct rcu_string *name;
922         u64 found_transid = btrfs_super_generation(disk_super);
923         u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
924         bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
925                 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
926         bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
927                                         BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
928
929         if (fsid_change_in_progress) {
930                 if (!has_metadata_uuid) {
931                         /*
932                          * When we have an image which has CHANGING_FSID_V2 set
933                          * it might belong to either a filesystem which has
934                          * disks with completed fsid change or it might belong
935                          * to fs with no UUID changes in effect, handle both.
936                          */
937                         fs_devices = find_fsid_inprogress(disk_super);
938                         if (!fs_devices)
939                                 fs_devices = find_fsid(disk_super->fsid, NULL);
940                 } else {
941                         fs_devices = find_fsid_changed(disk_super);
942                 }
943         } else if (has_metadata_uuid) {
944                 fs_devices = find_fsid(disk_super->fsid,
945                                        disk_super->metadata_uuid);
946         } else {
947                 fs_devices = find_fsid(disk_super->fsid, NULL);
948         }
949
950
951         if (!fs_devices) {
952                 if (has_metadata_uuid)
953                         fs_devices = alloc_fs_devices(disk_super->fsid,
954                                                       disk_super->metadata_uuid);
955                 else
956                         fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
957
958                 if (IS_ERR(fs_devices))
959                         return ERR_CAST(fs_devices);
960
961                 fs_devices->fsid_change = fsid_change_in_progress;
962
963                 mutex_lock(&fs_devices->device_list_mutex);
964                 list_add(&fs_devices->fs_list, &fs_uuids);
965
966                 device = NULL;
967         } else {
968                 mutex_lock(&fs_devices->device_list_mutex);
969                 device = btrfs_find_device(fs_devices, devid,
970                                 disk_super->dev_item.uuid, NULL, false);
971
972                 /*
973                  * If this disk has been pulled into an fs devices created by
974                  * a device which had the CHANGING_FSID_V2 flag then replace the
975                  * metadata_uuid/fsid values of the fs_devices.
976                  */
977                 if (has_metadata_uuid && fs_devices->fsid_change &&
978                     found_transid > fs_devices->latest_generation) {
979                         memcpy(fs_devices->fsid, disk_super->fsid,
980                                         BTRFS_FSID_SIZE);
981                         memcpy(fs_devices->metadata_uuid,
982                                         disk_super->metadata_uuid, BTRFS_FSID_SIZE);
983
984                         fs_devices->fsid_change = false;
985                 }
986         }
987
988         if (!device) {
989                 if (fs_devices->opened) {
990                         mutex_unlock(&fs_devices->device_list_mutex);
991                         return ERR_PTR(-EBUSY);
992                 }
993
994                 device = btrfs_alloc_device(NULL, &devid,
995                                             disk_super->dev_item.uuid);
996                 if (IS_ERR(device)) {
997                         mutex_unlock(&fs_devices->device_list_mutex);
998                         /* we can safely leave the fs_devices entry around */
999                         return device;
1000                 }
1001
1002                 name = rcu_string_strdup(path, GFP_NOFS);
1003                 if (!name) {
1004                         btrfs_free_device(device);
1005                         mutex_unlock(&fs_devices->device_list_mutex);
1006                         return ERR_PTR(-ENOMEM);
1007                 }
1008                 rcu_assign_pointer(device->name, name);
1009
1010                 list_add_rcu(&device->dev_list, &fs_devices->devices);
1011                 fs_devices->num_devices++;
1012
1013                 device->fs_devices = fs_devices;
1014                 *new_device_added = true;
1015
1016                 if (disk_super->label[0])
1017                         pr_info("BTRFS: device label %s devid %llu transid %llu %s\n",
1018                                 disk_super->label, devid, found_transid, path);
1019                 else
1020                         pr_info("BTRFS: device fsid %pU devid %llu transid %llu %s\n",
1021                                 disk_super->fsid, devid, found_transid, path);
1022
1023         } else if (!device->name || strcmp(device->name->str, path)) {
1024                 /*
1025                  * When FS is already mounted.
1026                  * 1. If you are here and if the device->name is NULL that
1027                  *    means this device was missing at time of FS mount.
1028                  * 2. If you are here and if the device->name is different
1029                  *    from 'path' that means either
1030                  *      a. The same device disappeared and reappeared with
1031                  *         different name. or
1032                  *      b. The missing-disk-which-was-replaced, has
1033                  *         reappeared now.
1034                  *
1035                  * We must allow 1 and 2a above. But 2b would be a spurious
1036                  * and unintentional.
1037                  *
1038                  * Further in case of 1 and 2a above, the disk at 'path'
1039                  * would have missed some transaction when it was away and
1040                  * in case of 2a the stale bdev has to be updated as well.
1041                  * 2b must not be allowed at all time.
1042                  */
1043
1044                 /*
1045                  * For now, we do allow update to btrfs_fs_device through the
1046                  * btrfs dev scan cli after FS has been mounted.  We're still
1047                  * tracking a problem where systems fail mount by subvolume id
1048                  * when we reject replacement on a mounted FS.
1049                  */
1050                 if (!fs_devices->opened && found_transid < device->generation) {
1051                         /*
1052                          * That is if the FS is _not_ mounted and if you
1053                          * are here, that means there is more than one
1054                          * disk with same uuid and devid.We keep the one
1055                          * with larger generation number or the last-in if
1056                          * generation are equal.
1057                          */
1058                         mutex_unlock(&fs_devices->device_list_mutex);
1059                         return ERR_PTR(-EEXIST);
1060                 }
1061
1062                 /*
1063                  * We are going to replace the device path for a given devid,
1064                  * make sure it's the same device if the device is mounted
1065                  */
1066                 if (device->bdev) {
1067                         struct block_device *path_bdev;
1068
1069                         path_bdev = lookup_bdev(path);
1070                         if (IS_ERR(path_bdev)) {
1071                                 mutex_unlock(&fs_devices->device_list_mutex);
1072                                 return ERR_CAST(path_bdev);
1073                         }
1074
1075                         if (device->bdev != path_bdev) {
1076                                 bdput(path_bdev);
1077                                 mutex_unlock(&fs_devices->device_list_mutex);
1078                                 btrfs_warn_in_rcu(device->fs_info,
1079                         "duplicate device fsid:devid for %pU:%llu old:%s new:%s",
1080                                         disk_super->fsid, devid,
1081                                         rcu_str_deref(device->name), path);
1082                                 return ERR_PTR(-EEXIST);
1083                         }
1084                         bdput(path_bdev);
1085                         btrfs_info_in_rcu(device->fs_info,
1086                                 "device fsid %pU devid %llu moved old:%s new:%s",
1087                                 disk_super->fsid, devid,
1088                                 rcu_str_deref(device->name), path);
1089                 }
1090
1091                 name = rcu_string_strdup(path, GFP_NOFS);
1092                 if (!name) {
1093                         mutex_unlock(&fs_devices->device_list_mutex);
1094                         return ERR_PTR(-ENOMEM);
1095                 }
1096                 rcu_string_free(device->name);
1097                 rcu_assign_pointer(device->name, name);
1098                 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1099                         fs_devices->missing_devices--;
1100                         clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1101                 }
1102         }
1103
1104         /*
1105          * Unmount does not free the btrfs_device struct but would zero
1106          * generation along with most of the other members. So just update
1107          * it back. We need it to pick the disk with largest generation
1108          * (as above).
1109          */
1110         if (!fs_devices->opened) {
1111                 device->generation = found_transid;
1112                 fs_devices->latest_generation = max_t(u64, found_transid,
1113                                                 fs_devices->latest_generation);
1114         }
1115
1116         fs_devices->total_devices = btrfs_super_num_devices(disk_super);
1117
1118         mutex_unlock(&fs_devices->device_list_mutex);
1119         return device;
1120 }
1121
1122 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
1123 {
1124         struct btrfs_fs_devices *fs_devices;
1125         struct btrfs_device *device;
1126         struct btrfs_device *orig_dev;
1127
1128         fs_devices = alloc_fs_devices(orig->fsid, NULL);
1129         if (IS_ERR(fs_devices))
1130                 return fs_devices;
1131
1132         mutex_lock(&orig->device_list_mutex);
1133         fs_devices->total_devices = orig->total_devices;
1134
1135         list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1136                 struct rcu_string *name;
1137
1138                 device = btrfs_alloc_device(NULL, &orig_dev->devid,
1139                                             orig_dev->uuid);
1140                 if (IS_ERR(device))
1141                         goto error;
1142
1143                 /*
1144                  * This is ok to do without rcu read locked because we hold the
1145                  * uuid mutex so nothing we touch in here is going to disappear.
1146                  */
1147                 if (orig_dev->name) {
1148                         name = rcu_string_strdup(orig_dev->name->str,
1149                                         GFP_KERNEL);
1150                         if (!name) {
1151                                 btrfs_free_device(device);
1152                                 goto error;
1153                         }
1154                         rcu_assign_pointer(device->name, name);
1155                 }
1156
1157                 list_add(&device->dev_list, &fs_devices->devices);
1158                 device->fs_devices = fs_devices;
1159                 fs_devices->num_devices++;
1160         }
1161         mutex_unlock(&orig->device_list_mutex);
1162         return fs_devices;
1163 error:
1164         mutex_unlock(&orig->device_list_mutex);
1165         free_fs_devices(fs_devices);
1166         return ERR_PTR(-ENOMEM);
1167 }
1168
1169 /*
1170  * After we have read the system tree and know devids belonging to
1171  * this filesystem, remove the device which does not belong there.
1172  */
1173 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
1174 {
1175         struct btrfs_device *device, *next;
1176         struct btrfs_device *latest_dev = NULL;
1177
1178         mutex_lock(&uuid_mutex);
1179 again:
1180         /* This is the initialized path, it is safe to release the devices. */
1181         list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1182                 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
1183                                                         &device->dev_state)) {
1184                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1185                              &device->dev_state) &&
1186                              (!latest_dev ||
1187                               device->generation > latest_dev->generation)) {
1188                                 latest_dev = device;
1189                         }
1190                         continue;
1191                 }
1192
1193                 if (device->devid == BTRFS_DEV_REPLACE_DEVID) {
1194                         /*
1195                          * In the first step, keep the device which has
1196                          * the correct fsid and the devid that is used
1197                          * for the dev_replace procedure.
1198                          * In the second step, the dev_replace state is
1199                          * read from the device tree and it is known
1200                          * whether the procedure is really active or
1201                          * not, which means whether this device is
1202                          * used or whether it should be removed.
1203                          */
1204                         if (step == 0 || test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1205                                                   &device->dev_state)) {
1206                                 continue;
1207                         }
1208                 }
1209                 if (device->bdev) {
1210                         blkdev_put(device->bdev, device->mode);
1211                         device->bdev = NULL;
1212                         fs_devices->open_devices--;
1213                 }
1214                 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1215                         list_del_init(&device->dev_alloc_list);
1216                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1217                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1218                                       &device->dev_state))
1219                                 fs_devices->rw_devices--;
1220                 }
1221                 list_del_init(&device->dev_list);
1222                 fs_devices->num_devices--;
1223                 btrfs_free_device(device);
1224         }
1225
1226         if (fs_devices->seed) {
1227                 fs_devices = fs_devices->seed;
1228                 goto again;
1229         }
1230
1231         fs_devices->latest_bdev = latest_dev->bdev;
1232
1233         mutex_unlock(&uuid_mutex);
1234 }
1235
1236 static void btrfs_close_bdev(struct btrfs_device *device)
1237 {
1238         if (!device->bdev)
1239                 return;
1240
1241         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1242                 sync_blockdev(device->bdev);
1243                 invalidate_bdev(device->bdev);
1244         }
1245
1246         blkdev_put(device->bdev, device->mode);
1247 }
1248
1249 static void btrfs_close_one_device(struct btrfs_device *device)
1250 {
1251         struct btrfs_fs_devices *fs_devices = device->fs_devices;
1252         struct btrfs_device *new_device;
1253         struct rcu_string *name;
1254
1255         if (device->bdev)
1256                 fs_devices->open_devices--;
1257
1258         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1259             device->devid != BTRFS_DEV_REPLACE_DEVID) {
1260                 list_del_init(&device->dev_alloc_list);
1261                 fs_devices->rw_devices--;
1262         }
1263
1264         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
1265                 fs_devices->missing_devices--;
1266
1267         btrfs_close_bdev(device);
1268
1269         new_device = btrfs_alloc_device(NULL, &device->devid,
1270                                         device->uuid);
1271         BUG_ON(IS_ERR(new_device)); /* -ENOMEM */
1272
1273         /* Safe because we are under uuid_mutex */
1274         if (device->name) {
1275                 name = rcu_string_strdup(device->name->str, GFP_NOFS);
1276                 BUG_ON(!name); /* -ENOMEM */
1277                 rcu_assign_pointer(new_device->name, name);
1278         }
1279
1280         list_replace_rcu(&device->dev_list, &new_device->dev_list);
1281         new_device->fs_devices = device->fs_devices;
1282
1283         synchronize_rcu();
1284         btrfs_free_device(device);
1285 }
1286
1287 static int close_fs_devices(struct btrfs_fs_devices *fs_devices)
1288 {
1289         struct btrfs_device *device, *tmp;
1290
1291         if (--fs_devices->opened > 0)
1292                 return 0;
1293
1294         mutex_lock(&fs_devices->device_list_mutex);
1295         list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) {
1296                 btrfs_close_one_device(device);
1297         }
1298         mutex_unlock(&fs_devices->device_list_mutex);
1299
1300         WARN_ON(fs_devices->open_devices);
1301         WARN_ON(fs_devices->rw_devices);
1302         fs_devices->opened = 0;
1303         fs_devices->seeding = 0;
1304
1305         return 0;
1306 }
1307
1308 int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1309 {
1310         struct btrfs_fs_devices *seed_devices = NULL;
1311         int ret;
1312
1313         mutex_lock(&uuid_mutex);
1314         ret = close_fs_devices(fs_devices);
1315         if (!fs_devices->opened) {
1316                 seed_devices = fs_devices->seed;
1317                 fs_devices->seed = NULL;
1318         }
1319         mutex_unlock(&uuid_mutex);
1320
1321         while (seed_devices) {
1322                 fs_devices = seed_devices;
1323                 seed_devices = fs_devices->seed;
1324                 close_fs_devices(fs_devices);
1325                 free_fs_devices(fs_devices);
1326         }
1327         return ret;
1328 }
1329
1330 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1331                                 fmode_t flags, void *holder)
1332 {
1333         struct btrfs_device *device;
1334         struct btrfs_device *latest_dev = NULL;
1335         int ret = 0;
1336
1337         flags |= FMODE_EXCL;
1338
1339         list_for_each_entry(device, &fs_devices->devices, dev_list) {
1340                 /* Just open everything we can; ignore failures here */
1341                 if (btrfs_open_one_device(fs_devices, device, flags, holder))
1342                         continue;
1343
1344                 if (!latest_dev ||
1345                     device->generation > latest_dev->generation)
1346                         latest_dev = device;
1347         }
1348         if (fs_devices->open_devices == 0) {
1349                 ret = -EINVAL;
1350                 goto out;
1351         }
1352         fs_devices->opened = 1;
1353         fs_devices->latest_bdev = latest_dev->bdev;
1354         fs_devices->total_rw_bytes = 0;
1355 out:
1356         return ret;
1357 }
1358
1359 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
1360 {
1361         struct btrfs_device *dev1, *dev2;
1362
1363         dev1 = list_entry(a, struct btrfs_device, dev_list);
1364         dev2 = list_entry(b, struct btrfs_device, dev_list);
1365
1366         if (dev1->devid < dev2->devid)
1367                 return -1;
1368         else if (dev1->devid > dev2->devid)
1369                 return 1;
1370         return 0;
1371 }
1372
1373 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1374                        fmode_t flags, void *holder)
1375 {
1376         int ret;
1377
1378         lockdep_assert_held(&uuid_mutex);
1379
1380         mutex_lock(&fs_devices->device_list_mutex);
1381         if (fs_devices->opened) {
1382                 fs_devices->opened++;
1383                 ret = 0;
1384         } else {
1385                 list_sort(NULL, &fs_devices->devices, devid_cmp);
1386                 ret = open_fs_devices(fs_devices, flags, holder);
1387         }
1388         mutex_unlock(&fs_devices->device_list_mutex);
1389
1390         return ret;
1391 }
1392
1393 static void btrfs_release_disk_super(struct page *page)
1394 {
1395         kunmap(page);
1396         put_page(page);
1397 }
1398
1399 static int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr,
1400                                  struct page **page,
1401                                  struct btrfs_super_block **disk_super)
1402 {
1403         void *p;
1404         pgoff_t index;
1405
1406         /* make sure our super fits in the device */
1407         if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1408                 return 1;
1409
1410         /* make sure our super fits in the page */
1411         if (sizeof(**disk_super) > PAGE_SIZE)
1412                 return 1;
1413
1414         /* make sure our super doesn't straddle pages on disk */
1415         index = bytenr >> PAGE_SHIFT;
1416         if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index)
1417                 return 1;
1418
1419         /* pull in the page with our super */
1420         *page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
1421                                    index, GFP_KERNEL);
1422
1423         if (IS_ERR_OR_NULL(*page))
1424                 return 1;
1425
1426         p = kmap(*page);
1427
1428         /* align our pointer to the offset of the super block */
1429         *disk_super = p + offset_in_page(bytenr);
1430
1431         if (btrfs_super_bytenr(*disk_super) != bytenr ||
1432             btrfs_super_magic(*disk_super) != BTRFS_MAGIC) {
1433                 btrfs_release_disk_super(*page);
1434                 return 1;
1435         }
1436
1437         if ((*disk_super)->label[0] &&
1438                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1])
1439                 (*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0';
1440
1441         return 0;
1442 }
1443
1444 int btrfs_forget_devices(const char *path)
1445 {
1446         int ret;
1447
1448         mutex_lock(&uuid_mutex);
1449         ret = btrfs_free_stale_devices(strlen(path) ? path : NULL, NULL);
1450         mutex_unlock(&uuid_mutex);
1451
1452         return ret;
1453 }
1454
1455 /*
1456  * Look for a btrfs signature on a device. This may be called out of the mount path
1457  * and we are not allowed to call set_blocksize during the scan. The superblock
1458  * is read via pagecache
1459  */
1460 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1461                                            void *holder)
1462 {
1463         struct btrfs_super_block *disk_super;
1464         bool new_device_added = false;
1465         struct btrfs_device *device = NULL;
1466         struct block_device *bdev;
1467         struct page *page;
1468         u64 bytenr;
1469
1470         lockdep_assert_held(&uuid_mutex);
1471
1472         /*
1473          * we would like to check all the supers, but that would make
1474          * a btrfs mount succeed after a mkfs from a different FS.
1475          * So, we need to add a special mount option to scan for
1476          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1477          */
1478         bytenr = btrfs_sb_offset(0);
1479         flags |= FMODE_EXCL;
1480
1481         bdev = blkdev_get_by_path(path, flags, holder);
1482         if (IS_ERR(bdev))
1483                 return ERR_CAST(bdev);
1484
1485         if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super)) {
1486                 device = ERR_PTR(-EINVAL);
1487                 goto error_bdev_put;
1488         }
1489
1490         device = device_list_add(path, disk_super, &new_device_added);
1491         if (!IS_ERR(device)) {
1492                 if (new_device_added)
1493                         btrfs_free_stale_devices(path, device);
1494         }
1495
1496         btrfs_release_disk_super(page);
1497
1498 error_bdev_put:
1499         blkdev_put(bdev, flags);
1500
1501         return device;
1502 }
1503
1504 /*
1505  * Try to find a chunk that intersects [start, start + len] range and when one
1506  * such is found, record the end of it in *start
1507  */
1508 static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1509                                     u64 len)
1510 {
1511         u64 physical_start, physical_end;
1512
1513         lockdep_assert_held(&device->fs_info->chunk_mutex);
1514
1515         if (!find_first_extent_bit(&device->alloc_state, *start,
1516                                    &physical_start, &physical_end,
1517                                    CHUNK_ALLOCATED, NULL)) {
1518
1519                 if (in_range(physical_start, *start, len) ||
1520                     in_range(*start, physical_start,
1521                              physical_end - physical_start)) {
1522                         *start = physical_end + 1;
1523                         return true;
1524                 }
1525         }
1526         return false;
1527 }
1528
1529
1530 /*
1531  * find_free_dev_extent_start - find free space in the specified device
1532  * @device:       the device which we search the free space in
1533  * @num_bytes:    the size of the free space that we need
1534  * @search_start: the position from which to begin the search
1535  * @start:        store the start of the free space.
1536  * @len:          the size of the free space. that we find, or the size
1537  *                of the max free space if we don't find suitable free space
1538  *
1539  * this uses a pretty simple search, the expectation is that it is
1540  * called very infrequently and that a given device has a small number
1541  * of extents
1542  *
1543  * @start is used to store the start of the free space if we find. But if we
1544  * don't find suitable free space, it will be used to store the start position
1545  * of the max free space.
1546  *
1547  * @len is used to store the size of the free space that we find.
1548  * But if we don't find suitable free space, it is used to store the size of
1549  * the max free space.
1550  */
1551 int find_free_dev_extent_start(struct btrfs_device *device, u64 num_bytes,
1552                                u64 search_start, u64 *start, u64 *len)
1553 {
1554         struct btrfs_fs_info *fs_info = device->fs_info;
1555         struct btrfs_root *root = fs_info->dev_root;
1556         struct btrfs_key key;
1557         struct btrfs_dev_extent *dev_extent;
1558         struct btrfs_path *path;
1559         u64 hole_size;
1560         u64 max_hole_start;
1561         u64 max_hole_size;
1562         u64 extent_end;
1563         u64 search_end = device->total_bytes;
1564         int ret;
1565         int slot;
1566         struct extent_buffer *l;
1567
1568         /*
1569          * We don't want to overwrite the superblock on the drive nor any area
1570          * used by the boot loader (grub for example), so we make sure to start
1571          * at an offset of at least 1MB.
1572          */
1573         search_start = max_t(u64, search_start, SZ_1M);
1574
1575         path = btrfs_alloc_path();
1576         if (!path)
1577                 return -ENOMEM;
1578
1579         max_hole_start = search_start;
1580         max_hole_size = 0;
1581
1582 again:
1583         if (search_start >= search_end ||
1584                 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1585                 ret = -ENOSPC;
1586                 goto out;
1587         }
1588
1589         path->reada = READA_FORWARD;
1590         path->search_commit_root = 1;
1591         path->skip_locking = 1;
1592
1593         key.objectid = device->devid;
1594         key.offset = search_start;
1595         key.type = BTRFS_DEV_EXTENT_KEY;
1596
1597         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1598         if (ret < 0)
1599                 goto out;
1600         if (ret > 0) {
1601                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1602                 if (ret < 0)
1603                         goto out;
1604         }
1605
1606         while (1) {
1607                 l = path->nodes[0];
1608                 slot = path->slots[0];
1609                 if (slot >= btrfs_header_nritems(l)) {
1610                         ret = btrfs_next_leaf(root, path);
1611                         if (ret == 0)
1612                                 continue;
1613                         if (ret < 0)
1614                                 goto out;
1615
1616                         break;
1617                 }
1618                 btrfs_item_key_to_cpu(l, &key, slot);
1619
1620                 if (key.objectid < device->devid)
1621                         goto next;
1622
1623                 if (key.objectid > device->devid)
1624                         break;
1625
1626                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1627                         goto next;
1628
1629                 if (key.offset > search_start) {
1630                         hole_size = key.offset - search_start;
1631
1632                         /*
1633                          * Have to check before we set max_hole_start, otherwise
1634                          * we could end up sending back this offset anyway.
1635                          */
1636                         if (contains_pending_extent(device, &search_start,
1637                                                     hole_size)) {
1638                                 if (key.offset >= search_start)
1639                                         hole_size = key.offset - search_start;
1640                                 else
1641                                         hole_size = 0;
1642                         }
1643
1644                         if (hole_size > max_hole_size) {
1645                                 max_hole_start = search_start;
1646                                 max_hole_size = hole_size;
1647                         }
1648
1649                         /*
1650                          * If this free space is greater than which we need,
1651                          * it must be the max free space that we have found
1652                          * until now, so max_hole_start must point to the start
1653                          * of this free space and the length of this free space
1654                          * is stored in max_hole_size. Thus, we return
1655                          * max_hole_start and max_hole_size and go back to the
1656                          * caller.
1657                          */
1658                         if (hole_size >= num_bytes) {
1659                                 ret = 0;
1660                                 goto out;
1661                         }
1662                 }
1663
1664                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1665                 extent_end = key.offset + btrfs_dev_extent_length(l,
1666                                                                   dev_extent);
1667                 if (extent_end > search_start)
1668                         search_start = extent_end;
1669 next:
1670                 path->slots[0]++;
1671                 cond_resched();
1672         }
1673
1674         /*
1675          * At this point, search_start should be the end of
1676          * allocated dev extents, and when shrinking the device,
1677          * search_end may be smaller than search_start.
1678          */
1679         if (search_end > search_start) {
1680                 hole_size = search_end - search_start;
1681
1682                 if (contains_pending_extent(device, &search_start, hole_size)) {
1683                         btrfs_release_path(path);
1684                         goto again;
1685                 }
1686
1687                 if (hole_size > max_hole_size) {
1688                         max_hole_start = search_start;
1689                         max_hole_size = hole_size;
1690                 }
1691         }
1692
1693         /* See above. */
1694         if (max_hole_size < num_bytes)
1695                 ret = -ENOSPC;
1696         else
1697                 ret = 0;
1698
1699 out:
1700         btrfs_free_path(path);
1701         *start = max_hole_start;
1702         if (len)
1703                 *len = max_hole_size;
1704         return ret;
1705 }
1706
1707 int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1708                          u64 *start, u64 *len)
1709 {
1710         /* FIXME use last free of some kind */
1711         return find_free_dev_extent_start(device, num_bytes, 0, start, len);
1712 }
1713
1714 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1715                           struct btrfs_device *device,
1716                           u64 start, u64 *dev_extent_len)
1717 {
1718         struct btrfs_fs_info *fs_info = device->fs_info;
1719         struct btrfs_root *root = fs_info->dev_root;
1720         int ret;
1721         struct btrfs_path *path;
1722         struct btrfs_key key;
1723         struct btrfs_key found_key;
1724         struct extent_buffer *leaf = NULL;
1725         struct btrfs_dev_extent *extent = NULL;
1726
1727         path = btrfs_alloc_path();
1728         if (!path)
1729                 return -ENOMEM;
1730
1731         key.objectid = device->devid;
1732         key.offset = start;
1733         key.type = BTRFS_DEV_EXTENT_KEY;
1734 again:
1735         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1736         if (ret > 0) {
1737                 ret = btrfs_previous_item(root, path, key.objectid,
1738                                           BTRFS_DEV_EXTENT_KEY);
1739                 if (ret)
1740                         goto out;
1741                 leaf = path->nodes[0];
1742                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1743                 extent = btrfs_item_ptr(leaf, path->slots[0],
1744                                         struct btrfs_dev_extent);
1745                 BUG_ON(found_key.offset > start || found_key.offset +
1746                        btrfs_dev_extent_length(leaf, extent) < start);
1747                 key = found_key;
1748                 btrfs_release_path(path);
1749                 goto again;
1750         } else if (ret == 0) {
1751                 leaf = path->nodes[0];
1752                 extent = btrfs_item_ptr(leaf, path->slots[0],
1753                                         struct btrfs_dev_extent);
1754         } else {
1755                 btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1756                 goto out;
1757         }
1758
1759         *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1760
1761         ret = btrfs_del_item(trans, root, path);
1762         if (ret) {
1763                 btrfs_handle_fs_error(fs_info, ret,
1764                                       "Failed to remove dev extent item");
1765         } else {
1766                 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1767         }
1768 out:
1769         btrfs_free_path(path);
1770         return ret;
1771 }
1772
1773 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1774                                   struct btrfs_device *device,
1775                                   u64 chunk_offset, u64 start, u64 num_bytes)
1776 {
1777         int ret;
1778         struct btrfs_path *path;
1779         struct btrfs_fs_info *fs_info = device->fs_info;
1780         struct btrfs_root *root = fs_info->dev_root;
1781         struct btrfs_dev_extent *extent;
1782         struct extent_buffer *leaf;
1783         struct btrfs_key key;
1784
1785         WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state));
1786         WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1787         path = btrfs_alloc_path();
1788         if (!path)
1789                 return -ENOMEM;
1790
1791         key.objectid = device->devid;
1792         key.offset = start;
1793         key.type = BTRFS_DEV_EXTENT_KEY;
1794         ret = btrfs_insert_empty_item(trans, root, path, &key,
1795                                       sizeof(*extent));
1796         if (ret)
1797                 goto out;
1798
1799         leaf = path->nodes[0];
1800         extent = btrfs_item_ptr(leaf, path->slots[0],
1801                                 struct btrfs_dev_extent);
1802         btrfs_set_dev_extent_chunk_tree(leaf, extent,
1803                                         BTRFS_CHUNK_TREE_OBJECTID);
1804         btrfs_set_dev_extent_chunk_objectid(leaf, extent,
1805                                             BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1806         btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1807
1808         btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1809         btrfs_mark_buffer_dirty(leaf);
1810 out:
1811         btrfs_free_path(path);
1812         return ret;
1813 }
1814
1815 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1816 {
1817         struct extent_map_tree *em_tree;
1818         struct extent_map *em;
1819         struct rb_node *n;
1820         u64 ret = 0;
1821
1822         em_tree = &fs_info->mapping_tree.map_tree;
1823         read_lock(&em_tree->lock);
1824         n = rb_last(&em_tree->map.rb_root);
1825         if (n) {
1826                 em = rb_entry(n, struct extent_map, rb_node);
1827                 ret = em->start + em->len;
1828         }
1829         read_unlock(&em_tree->lock);
1830
1831         return ret;
1832 }
1833
1834 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1835                                     u64 *devid_ret)
1836 {
1837         int ret;
1838         struct btrfs_key key;
1839         struct btrfs_key found_key;
1840         struct btrfs_path *path;
1841
1842         path = btrfs_alloc_path();
1843         if (!path)
1844                 return -ENOMEM;
1845
1846         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1847         key.type = BTRFS_DEV_ITEM_KEY;
1848         key.offset = (u64)-1;
1849
1850         ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1851         if (ret < 0)
1852                 goto error;
1853
1854         BUG_ON(ret == 0); /* Corruption */
1855
1856         ret = btrfs_previous_item(fs_info->chunk_root, path,
1857                                   BTRFS_DEV_ITEMS_OBJECTID,
1858                                   BTRFS_DEV_ITEM_KEY);
1859         if (ret) {
1860                 *devid_ret = 1;
1861         } else {
1862                 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1863                                       path->slots[0]);
1864                 *devid_ret = found_key.offset + 1;
1865         }
1866         ret = 0;
1867 error:
1868         btrfs_free_path(path);
1869         return ret;
1870 }
1871
1872 /*
1873  * the device information is stored in the chunk root
1874  * the btrfs_device struct should be fully filled in
1875  */
1876 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1877                             struct btrfs_device *device)
1878 {
1879         int ret;
1880         struct btrfs_path *path;
1881         struct btrfs_dev_item *dev_item;
1882         struct extent_buffer *leaf;
1883         struct btrfs_key key;
1884         unsigned long ptr;
1885
1886         path = btrfs_alloc_path();
1887         if (!path)
1888                 return -ENOMEM;
1889
1890         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1891         key.type = BTRFS_DEV_ITEM_KEY;
1892         key.offset = device->devid;
1893
1894         ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1895                                       &key, sizeof(*dev_item));
1896         if (ret)
1897                 goto out;
1898
1899         leaf = path->nodes[0];
1900         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1901
1902         btrfs_set_device_id(leaf, dev_item, device->devid);
1903         btrfs_set_device_generation(leaf, dev_item, 0);
1904         btrfs_set_device_type(leaf, dev_item, device->type);
1905         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1906         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1907         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1908         btrfs_set_device_total_bytes(leaf, dev_item,
1909                                      btrfs_device_get_disk_total_bytes(device));
1910         btrfs_set_device_bytes_used(leaf, dev_item,
1911                                     btrfs_device_get_bytes_used(device));
1912         btrfs_set_device_group(leaf, dev_item, 0);
1913         btrfs_set_device_seek_speed(leaf, dev_item, 0);
1914         btrfs_set_device_bandwidth(leaf, dev_item, 0);
1915         btrfs_set_device_start_offset(leaf, dev_item, 0);
1916
1917         ptr = btrfs_device_uuid(dev_item);
1918         write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1919         ptr = btrfs_device_fsid(dev_item);
1920         write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1921                             ptr, BTRFS_FSID_SIZE);
1922         btrfs_mark_buffer_dirty(leaf);
1923
1924         ret = 0;
1925 out:
1926         btrfs_free_path(path);
1927         return ret;
1928 }
1929
1930 /*
1931  * Function to update ctime/mtime for a given device path.
1932  * Mainly used for ctime/mtime based probe like libblkid.
1933  */
1934 static void update_dev_time(const char *path_name)
1935 {
1936         struct file *filp;
1937
1938         filp = filp_open(path_name, O_RDWR, 0);
1939         if (IS_ERR(filp))
1940                 return;
1941         file_update_time(filp);
1942         filp_close(filp, NULL);
1943 }
1944
1945 static int btrfs_rm_dev_item(struct btrfs_fs_info *fs_info,
1946                              struct btrfs_device *device)
1947 {
1948         struct btrfs_root *root = fs_info->chunk_root;
1949         int ret;
1950         struct btrfs_path *path;
1951         struct btrfs_key key;
1952         struct btrfs_trans_handle *trans;
1953
1954         path = btrfs_alloc_path();
1955         if (!path)
1956                 return -ENOMEM;
1957
1958         trans = btrfs_start_transaction(root, 0);
1959         if (IS_ERR(trans)) {
1960                 btrfs_free_path(path);
1961                 return PTR_ERR(trans);
1962         }
1963         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1964         key.type = BTRFS_DEV_ITEM_KEY;
1965         key.offset = device->devid;
1966
1967         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1968         if (ret) {
1969                 if (ret > 0)
1970                         ret = -ENOENT;
1971                 btrfs_abort_transaction(trans, ret);
1972                 btrfs_end_transaction(trans);
1973                 goto out;
1974         }
1975
1976         ret = btrfs_del_item(trans, root, path);
1977         if (ret) {
1978                 btrfs_abort_transaction(trans, ret);
1979                 btrfs_end_transaction(trans);
1980         }
1981
1982 out:
1983         btrfs_free_path(path);
1984         if (!ret)
1985                 ret = btrfs_commit_transaction(trans);
1986         return ret;
1987 }
1988
1989 /*
1990  * Verify that @num_devices satisfies the RAID profile constraints in the whole
1991  * filesystem. It's up to the caller to adjust that number regarding eg. device
1992  * replace.
1993  */
1994 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1995                 u64 num_devices)
1996 {
1997         u64 all_avail;
1998         unsigned seq;
1999         int i;
2000
2001         do {
2002                 seq = read_seqbegin(&fs_info->profiles_lock);
2003
2004                 all_avail = fs_info->avail_data_alloc_bits |
2005                             fs_info->avail_system_alloc_bits |
2006                             fs_info->avail_metadata_alloc_bits;
2007         } while (read_seqretry(&fs_info->profiles_lock, seq));
2008
2009         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2010                 if (!(all_avail & btrfs_raid_array[i].bg_flag))
2011                         continue;
2012
2013                 if (num_devices < btrfs_raid_array[i].devs_min) {
2014                         int ret = btrfs_raid_array[i].mindev_error;
2015
2016                         if (ret)
2017                                 return ret;
2018                 }
2019         }
2020
2021         return 0;
2022 }
2023
2024 static struct btrfs_device * btrfs_find_next_active_device(
2025                 struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2026 {
2027         struct btrfs_device *next_device;
2028
2029         list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2030                 if (next_device != device &&
2031                     !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2032                     && next_device->bdev)
2033                         return next_device;
2034         }
2035
2036         return NULL;
2037 }
2038
2039 /*
2040  * Helper function to check if the given device is part of s_bdev / latest_bdev
2041  * and replace it with the provided or the next active device, in the context
2042  * where this function called, there should be always be another device (or
2043  * this_dev) which is active.
2044  */
2045 void btrfs_assign_next_active_device(struct btrfs_device *device,
2046                                      struct btrfs_device *this_dev)
2047 {
2048         struct btrfs_fs_info *fs_info = device->fs_info;
2049         struct btrfs_device *next_device;
2050
2051         if (this_dev)
2052                 next_device = this_dev;
2053         else
2054                 next_device = btrfs_find_next_active_device(fs_info->fs_devices,
2055                                                                 device);
2056         ASSERT(next_device);
2057
2058         if (fs_info->sb->s_bdev &&
2059                         (fs_info->sb->s_bdev == device->bdev))
2060                 fs_info->sb->s_bdev = next_device->bdev;
2061
2062         if (fs_info->fs_devices->latest_bdev == device->bdev)
2063                 fs_info->fs_devices->latest_bdev = next_device->bdev;
2064 }
2065
2066 /*
2067  * Return btrfs_fs_devices::num_devices excluding the device that's being
2068  * currently replaced.
2069  */
2070 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2071 {
2072         u64 num_devices = fs_info->fs_devices->num_devices;
2073
2074         down_read(&fs_info->dev_replace.rwsem);
2075         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
2076                 ASSERT(num_devices > 1);
2077                 num_devices--;
2078         }
2079         up_read(&fs_info->dev_replace.rwsem);
2080
2081         return num_devices;
2082 }
2083
2084 int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
2085                 u64 devid)
2086 {
2087         struct btrfs_device *device;
2088         struct btrfs_fs_devices *cur_devices;
2089         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2090         u64 num_devices;
2091         int ret = 0;
2092
2093         mutex_lock(&uuid_mutex);
2094
2095         num_devices = btrfs_num_devices(fs_info);
2096
2097         ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2098         if (ret)
2099                 goto out;
2100
2101         device = btrfs_find_device_by_devspec(fs_info, devid, device_path);
2102
2103         if (IS_ERR(device)) {
2104                 if (PTR_ERR(device) == -ENOENT &&
2105                     strcmp(device_path, "missing") == 0)
2106                         ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2107                 else
2108                         ret = PTR_ERR(device);
2109                 goto out;
2110         }
2111
2112         if (btrfs_pinned_by_swapfile(fs_info, device)) {
2113                 btrfs_warn_in_rcu(fs_info,
2114                   "cannot remove device %s (devid %llu) due to active swapfile",
2115                                   rcu_str_deref(device->name), device->devid);
2116                 ret = -ETXTBSY;
2117                 goto out;
2118         }
2119
2120         if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2121                 ret = BTRFS_ERROR_DEV_TGT_REPLACE;
2122                 goto out;
2123         }
2124
2125         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2126             fs_info->fs_devices->rw_devices == 1) {
2127                 ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
2128                 goto out;
2129         }
2130
2131         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2132                 mutex_lock(&fs_info->chunk_mutex);
2133                 list_del_init(&device->dev_alloc_list);
2134                 device->fs_devices->rw_devices--;
2135                 mutex_unlock(&fs_info->chunk_mutex);
2136         }
2137
2138         mutex_unlock(&uuid_mutex);
2139         ret = btrfs_shrink_device(device, 0);
2140         mutex_lock(&uuid_mutex);
2141         if (ret)
2142                 goto error_undo;
2143
2144         /*
2145          * TODO: the superblock still includes this device in its num_devices
2146          * counter although write_all_supers() is not locked out. This
2147          * could give a filesystem state which requires a degraded mount.
2148          */
2149         ret = btrfs_rm_dev_item(fs_info, device);
2150         if (ret)
2151                 goto error_undo;
2152
2153         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2154         btrfs_scrub_cancel_dev(fs_info, device);
2155
2156         /*
2157          * the device list mutex makes sure that we don't change
2158          * the device list while someone else is writing out all
2159          * the device supers. Whoever is writing all supers, should
2160          * lock the device list mutex before getting the number of
2161          * devices in the super block (super_copy). Conversely,
2162          * whoever updates the number of devices in the super block
2163          * (super_copy) should hold the device list mutex.
2164          */
2165
2166         /*
2167          * In normal cases the cur_devices == fs_devices. But in case
2168          * of deleting a seed device, the cur_devices should point to
2169          * its own fs_devices listed under the fs_devices->seed.
2170          */
2171         cur_devices = device->fs_devices;
2172         mutex_lock(&fs_devices->device_list_mutex);
2173         list_del_rcu(&device->dev_list);
2174
2175         cur_devices->num_devices--;
2176         cur_devices->total_devices--;
2177         /* Update total_devices of the parent fs_devices if it's seed */
2178         if (cur_devices != fs_devices)
2179                 fs_devices->total_devices--;
2180
2181         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2182                 cur_devices->missing_devices--;
2183
2184         btrfs_assign_next_active_device(device, NULL);
2185
2186         if (device->bdev) {
2187                 cur_devices->open_devices--;
2188                 /* remove sysfs entry */
2189                 btrfs_sysfs_rm_device_link(fs_devices, device);
2190         }
2191
2192         num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2193         btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2194         mutex_unlock(&fs_devices->device_list_mutex);
2195
2196         /*
2197          * at this point, the device is zero sized and detached from
2198          * the devices list.  All that's left is to zero out the old
2199          * supers and free the device.
2200          */
2201         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2202                 btrfs_scratch_superblocks(device->bdev, device->name->str);
2203
2204         btrfs_close_bdev(device);
2205         synchronize_rcu();
2206         btrfs_free_device(device);
2207
2208         if (cur_devices->open_devices == 0) {
2209                 while (fs_devices) {
2210                         if (fs_devices->seed == cur_devices) {
2211                                 fs_devices->seed = cur_devices->seed;
2212                                 break;
2213                         }
2214                         fs_devices = fs_devices->seed;
2215                 }
2216                 cur_devices->seed = NULL;
2217                 close_fs_devices(cur_devices);
2218                 free_fs_devices(cur_devices);
2219         }
2220
2221 out:
2222         mutex_unlock(&uuid_mutex);
2223         return ret;
2224
2225 error_undo:
2226         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2227                 mutex_lock(&fs_info->chunk_mutex);
2228                 list_add(&device->dev_alloc_list,
2229                          &fs_devices->alloc_list);
2230                 device->fs_devices->rw_devices++;
2231                 mutex_unlock(&fs_info->chunk_mutex);
2232         }
2233         goto out;
2234 }
2235
2236 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2237 {
2238         struct btrfs_fs_devices *fs_devices;
2239
2240         lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2241
2242         /*
2243          * in case of fs with no seed, srcdev->fs_devices will point
2244          * to fs_devices of fs_info. However when the dev being replaced is
2245          * a seed dev it will point to the seed's local fs_devices. In short
2246          * srcdev will have its correct fs_devices in both the cases.
2247          */
2248         fs_devices = srcdev->fs_devices;
2249
2250         list_del_rcu(&srcdev->dev_list);
2251         list_del(&srcdev->dev_alloc_list);
2252         fs_devices->num_devices--;
2253         if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2254                 fs_devices->missing_devices--;
2255
2256         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2257                 fs_devices->rw_devices--;
2258
2259         if (srcdev->bdev)
2260                 fs_devices->open_devices--;
2261 }
2262
2263 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_fs_info *fs_info,
2264                                       struct btrfs_device *srcdev)
2265 {
2266         struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2267
2268         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) {
2269                 /* zero out the old super if it is writable */
2270                 btrfs_scratch_superblocks(srcdev->bdev, srcdev->name->str);
2271         }
2272
2273         btrfs_close_bdev(srcdev);
2274         synchronize_rcu();
2275         btrfs_free_device(srcdev);
2276
2277         /* if this is no devs we rather delete the fs_devices */
2278         if (!fs_devices->num_devices) {
2279                 struct btrfs_fs_devices *tmp_fs_devices;
2280
2281                 /*
2282                  * On a mounted FS, num_devices can't be zero unless it's a
2283                  * seed. In case of a seed device being replaced, the replace
2284                  * target added to the sprout FS, so there will be no more
2285                  * device left under the seed FS.
2286                  */
2287                 ASSERT(fs_devices->seeding);
2288
2289                 tmp_fs_devices = fs_info->fs_devices;
2290                 while (tmp_fs_devices) {
2291                         if (tmp_fs_devices->seed == fs_devices) {
2292                                 tmp_fs_devices->seed = fs_devices->seed;
2293                                 break;
2294                         }
2295                         tmp_fs_devices = tmp_fs_devices->seed;
2296                 }
2297                 fs_devices->seed = NULL;
2298                 close_fs_devices(fs_devices);
2299                 free_fs_devices(fs_devices);
2300         }
2301 }
2302
2303 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2304 {
2305         struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2306
2307         WARN_ON(!tgtdev);
2308         mutex_lock(&fs_devices->device_list_mutex);
2309
2310         btrfs_sysfs_rm_device_link(fs_devices, tgtdev);
2311
2312         if (tgtdev->bdev)
2313                 fs_devices->open_devices--;
2314
2315         fs_devices->num_devices--;
2316
2317         btrfs_assign_next_active_device(tgtdev, NULL);
2318
2319         list_del_rcu(&tgtdev->dev_list);
2320
2321         mutex_unlock(&fs_devices->device_list_mutex);
2322
2323         /*
2324          * The update_dev_time() with in btrfs_scratch_superblocks()
2325          * may lead to a call to btrfs_show_devname() which will try
2326          * to hold device_list_mutex. And here this device
2327          * is already out of device list, so we don't have to hold
2328          * the device_list_mutex lock.
2329          */
2330         btrfs_scratch_superblocks(tgtdev->bdev, tgtdev->name->str);
2331
2332         btrfs_close_bdev(tgtdev);
2333         synchronize_rcu();
2334         btrfs_free_device(tgtdev);
2335 }
2336
2337 static struct btrfs_device *btrfs_find_device_by_path(
2338                 struct btrfs_fs_info *fs_info, const char *device_path)
2339 {
2340         int ret = 0;
2341         struct btrfs_super_block *disk_super;
2342         u64 devid;
2343         u8 *dev_uuid;
2344         struct block_device *bdev;
2345         struct buffer_head *bh;
2346         struct btrfs_device *device;
2347
2348         ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2349                                     fs_info->bdev_holder, 0, &bdev, &bh);
2350         if (ret)
2351                 return ERR_PTR(ret);
2352         disk_super = (struct btrfs_super_block *)bh->b_data;
2353         devid = btrfs_stack_device_id(&disk_super->dev_item);
2354         dev_uuid = disk_super->dev_item.uuid;
2355         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2356                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2357                                            disk_super->metadata_uuid, true);
2358         else
2359                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2360                                            disk_super->fsid, true);
2361
2362         brelse(bh);
2363         if (!device)
2364                 device = ERR_PTR(-ENOENT);
2365         blkdev_put(bdev, FMODE_READ);
2366         return device;
2367 }
2368
2369 /*
2370  * Lookup a device given by device id, or the path if the id is 0.
2371  */
2372 struct btrfs_device *btrfs_find_device_by_devspec(
2373                 struct btrfs_fs_info *fs_info, u64 devid,
2374                 const char *device_path)
2375 {
2376         struct btrfs_device *device;
2377
2378         if (devid) {
2379                 device = btrfs_find_device(fs_info->fs_devices, devid, NULL,
2380                                            NULL, true);
2381                 if (!device)
2382                         return ERR_PTR(-ENOENT);
2383                 return device;
2384         }
2385
2386         if (!device_path || !device_path[0])
2387                 return ERR_PTR(-EINVAL);
2388
2389         if (strcmp(device_path, "missing") == 0) {
2390                 /* Find first missing device */
2391                 list_for_each_entry(device, &fs_info->fs_devices->devices,
2392                                     dev_list) {
2393                         if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
2394                                      &device->dev_state) && !device->bdev)
2395                                 return device;
2396                 }
2397                 return ERR_PTR(-ENOENT);
2398         }
2399
2400         return btrfs_find_device_by_path(fs_info, device_path);
2401 }
2402
2403 /*
2404  * does all the dirty work required for changing file system's UUID.
2405  */
2406 static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
2407 {
2408         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2409         struct btrfs_fs_devices *old_devices;
2410         struct btrfs_fs_devices *seed_devices;
2411         struct btrfs_super_block *disk_super = fs_info->super_copy;
2412         struct btrfs_device *device;
2413         u64 super_flags;
2414
2415         lockdep_assert_held(&uuid_mutex);
2416         if (!fs_devices->seeding)
2417                 return -EINVAL;
2418
2419         seed_devices = alloc_fs_devices(NULL, NULL);
2420         if (IS_ERR(seed_devices))
2421                 return PTR_ERR(seed_devices);
2422
2423         old_devices = clone_fs_devices(fs_devices);
2424         if (IS_ERR(old_devices)) {
2425                 kfree(seed_devices);
2426                 return PTR_ERR(old_devices);
2427         }
2428
2429         list_add(&old_devices->fs_list, &fs_uuids);
2430
2431         memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2432         seed_devices->opened = 1;
2433         INIT_LIST_HEAD(&seed_devices->devices);
2434         INIT_LIST_HEAD(&seed_devices->alloc_list);
2435         mutex_init(&seed_devices->device_list_mutex);
2436
2437         mutex_lock(&fs_devices->device_list_mutex);
2438         list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2439                               synchronize_rcu);
2440         list_for_each_entry(device, &seed_devices->devices, dev_list)
2441                 device->fs_devices = seed_devices;
2442
2443         mutex_lock(&fs_info->chunk_mutex);
2444         list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
2445         mutex_unlock(&fs_info->chunk_mutex);
2446
2447         fs_devices->seeding = 0;
2448         fs_devices->num_devices = 0;
2449         fs_devices->open_devices = 0;
2450         fs_devices->missing_devices = 0;
2451         fs_devices->rotating = 0;
2452         fs_devices->seed = seed_devices;
2453
2454         generate_random_uuid(fs_devices->fsid);
2455         memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2456         memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2457         mutex_unlock(&fs_devices->device_list_mutex);
2458
2459         super_flags = btrfs_super_flags(disk_super) &
2460                       ~BTRFS_SUPER_FLAG_SEEDING;
2461         btrfs_set_super_flags(disk_super, super_flags);
2462
2463         return 0;
2464 }
2465
2466 /*
2467  * Store the expected generation for seed devices in device items.
2468  */
2469 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans,
2470                                struct btrfs_fs_info *fs_info)
2471 {
2472         struct btrfs_root *root = fs_info->chunk_root;
2473         struct btrfs_path *path;
2474         struct extent_buffer *leaf;
2475         struct btrfs_dev_item *dev_item;
2476         struct btrfs_device *device;
2477         struct btrfs_key key;
2478         u8 fs_uuid[BTRFS_FSID_SIZE];
2479         u8 dev_uuid[BTRFS_UUID_SIZE];
2480         u64 devid;
2481         int ret;
2482
2483         path = btrfs_alloc_path();
2484         if (!path)
2485                 return -ENOMEM;
2486
2487         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2488         key.offset = 0;
2489         key.type = BTRFS_DEV_ITEM_KEY;
2490
2491         while (1) {
2492                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2493                 if (ret < 0)
2494                         goto error;
2495
2496                 leaf = path->nodes[0];
2497 next_slot:
2498                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2499                         ret = btrfs_next_leaf(root, path);
2500                         if (ret > 0)
2501                                 break;
2502                         if (ret < 0)
2503                                 goto error;
2504                         leaf = path->nodes[0];
2505                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2506                         btrfs_release_path(path);
2507                         continue;
2508                 }
2509
2510                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2511                 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2512                     key.type != BTRFS_DEV_ITEM_KEY)
2513                         break;
2514
2515                 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2516                                           struct btrfs_dev_item);
2517                 devid = btrfs_device_id(leaf, dev_item);
2518                 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2519                                    BTRFS_UUID_SIZE);
2520                 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2521                                    BTRFS_FSID_SIZE);
2522                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2523                                            fs_uuid, true);
2524                 BUG_ON(!device); /* Logic error */
2525
2526                 if (device->fs_devices->seeding) {
2527                         btrfs_set_device_generation(leaf, dev_item,
2528                                                     device->generation);
2529                         btrfs_mark_buffer_dirty(leaf);
2530                 }
2531
2532                 path->slots[0]++;
2533                 goto next_slot;
2534         }
2535         ret = 0;
2536 error:
2537         btrfs_free_path(path);
2538         return ret;
2539 }
2540
2541 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2542 {
2543         struct btrfs_root *root = fs_info->dev_root;
2544         struct request_queue *q;
2545         struct btrfs_trans_handle *trans;
2546         struct btrfs_device *device;
2547         struct block_device *bdev;
2548         struct super_block *sb = fs_info->sb;
2549         struct rcu_string *name;
2550         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2551         u64 orig_super_total_bytes;
2552         u64 orig_super_num_devices;
2553         int seeding_dev = 0;
2554         int ret = 0;
2555         bool unlocked = false;
2556
2557         if (sb_rdonly(sb) && !fs_devices->seeding)
2558                 return -EROFS;
2559
2560         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2561                                   fs_info->bdev_holder);
2562         if (IS_ERR(bdev))
2563                 return PTR_ERR(bdev);
2564
2565         if (fs_devices->seeding) {
2566                 seeding_dev = 1;
2567                 down_write(&sb->s_umount);
2568                 mutex_lock(&uuid_mutex);
2569         }
2570
2571         filemap_write_and_wait(bdev->bd_inode->i_mapping);
2572
2573         mutex_lock(&fs_devices->device_list_mutex);
2574         list_for_each_entry(device, &fs_devices->devices, dev_list) {
2575                 if (device->bdev == bdev) {
2576                         ret = -EEXIST;
2577                         mutex_unlock(
2578                                 &fs_devices->device_list_mutex);
2579                         goto error;
2580                 }
2581         }
2582         mutex_unlock(&fs_devices->device_list_mutex);
2583
2584         device = btrfs_alloc_device(fs_info, NULL, NULL);
2585         if (IS_ERR(device)) {
2586                 /* we can safely leave the fs_devices entry around */
2587                 ret = PTR_ERR(device);
2588                 goto error;
2589         }
2590
2591         name = rcu_string_strdup(device_path, GFP_KERNEL);
2592         if (!name) {
2593                 ret = -ENOMEM;
2594                 goto error_free_device;
2595         }
2596         rcu_assign_pointer(device->name, name);
2597
2598         trans = btrfs_start_transaction(root, 0);
2599         if (IS_ERR(trans)) {
2600                 ret = PTR_ERR(trans);
2601                 goto error_free_device;
2602         }
2603
2604         q = bdev_get_queue(bdev);
2605         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2606         device->generation = trans->transid;
2607         device->io_width = fs_info->sectorsize;
2608         device->io_align = fs_info->sectorsize;
2609         device->sector_size = fs_info->sectorsize;
2610         device->total_bytes = round_down(i_size_read(bdev->bd_inode),
2611                                          fs_info->sectorsize);
2612         device->disk_total_bytes = device->total_bytes;
2613         device->commit_total_bytes = device->total_bytes;
2614         device->fs_info = fs_info;
2615         device->bdev = bdev;
2616         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2617         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2618         device->mode = FMODE_EXCL;
2619         device->dev_stats_valid = 1;
2620         set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2621
2622         if (seeding_dev) {
2623                 sb->s_flags &= ~SB_RDONLY;
2624                 ret = btrfs_prepare_sprout(fs_info);
2625                 if (ret) {
2626                         btrfs_abort_transaction(trans, ret);
2627                         goto error_trans;
2628                 }
2629         }
2630
2631         device->fs_devices = fs_devices;
2632
2633         mutex_lock(&fs_devices->device_list_mutex);
2634         mutex_lock(&fs_info->chunk_mutex);
2635         list_add_rcu(&device->dev_list, &fs_devices->devices);
2636         list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2637         fs_devices->num_devices++;
2638         fs_devices->open_devices++;
2639         fs_devices->rw_devices++;
2640         fs_devices->total_devices++;
2641         fs_devices->total_rw_bytes += device->total_bytes;
2642
2643         atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2644
2645         if (!blk_queue_nonrot(q))
2646                 fs_devices->rotating = 1;
2647
2648         orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2649         btrfs_set_super_total_bytes(fs_info->super_copy,
2650                 round_down(orig_super_total_bytes + device->total_bytes,
2651                            fs_info->sectorsize));
2652
2653         orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2654         btrfs_set_super_num_devices(fs_info->super_copy,
2655                                     orig_super_num_devices + 1);
2656
2657         /* add sysfs device entry */
2658         btrfs_sysfs_add_device_link(fs_devices, device);
2659
2660         /*
2661          * we've got more storage, clear any full flags on the space
2662          * infos
2663          */
2664         btrfs_clear_space_info_full(fs_info);
2665
2666         mutex_unlock(&fs_info->chunk_mutex);
2667         mutex_unlock(&fs_devices->device_list_mutex);
2668
2669         if (seeding_dev) {
2670                 mutex_lock(&fs_info->chunk_mutex);
2671                 ret = init_first_rw_device(trans, fs_info);
2672                 mutex_unlock(&fs_info->chunk_mutex);
2673                 if (ret) {
2674                         btrfs_abort_transaction(trans, ret);
2675                         goto error_sysfs;
2676                 }
2677         }
2678
2679         ret = btrfs_add_dev_item(trans, device);
2680         if (ret) {
2681                 btrfs_abort_transaction(trans, ret);
2682                 goto error_sysfs;
2683         }
2684
2685         if (seeding_dev) {
2686                 char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
2687
2688                 ret = btrfs_finish_sprout(trans, fs_info);
2689                 if (ret) {
2690                         btrfs_abort_transaction(trans, ret);
2691                         goto error_sysfs;
2692                 }
2693
2694                 /* Sprouting would change fsid of the mounted root,
2695                  * so rename the fsid on the sysfs
2696                  */
2697                 snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
2698                                                 fs_info->fs_devices->fsid);
2699                 if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf))
2700                         btrfs_warn(fs_info,
2701                                    "sysfs: failed to create fsid for sprout");
2702         }
2703
2704         ret = btrfs_commit_transaction(trans);
2705
2706         if (seeding_dev) {
2707                 mutex_unlock(&uuid_mutex);
2708                 up_write(&sb->s_umount);
2709                 unlocked = true;
2710
2711                 if (ret) /* transaction commit */
2712                         return ret;
2713
2714                 ret = btrfs_relocate_sys_chunks(fs_info);
2715                 if (ret < 0)
2716                         btrfs_handle_fs_error(fs_info, ret,
2717                                     "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2718                 trans = btrfs_attach_transaction(root);
2719                 if (IS_ERR(trans)) {
2720                         if (PTR_ERR(trans) == -ENOENT)
2721                                 return 0;
2722                         ret = PTR_ERR(trans);
2723                         trans = NULL;
2724                         goto error_sysfs;
2725                 }
2726                 ret = btrfs_commit_transaction(trans);
2727         }
2728
2729         /* Update ctime/mtime for libblkid */
2730         update_dev_time(device_path);
2731         return ret;
2732
2733 error_sysfs:
2734         btrfs_sysfs_rm_device_link(fs_devices, device);
2735         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2736         mutex_lock(&fs_info->chunk_mutex);
2737         list_del_rcu(&device->dev_list);
2738         list_del(&device->dev_alloc_list);
2739         fs_info->fs_devices->num_devices--;
2740         fs_info->fs_devices->open_devices--;
2741         fs_info->fs_devices->rw_devices--;
2742         fs_info->fs_devices->total_devices--;
2743         fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2744         atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2745         btrfs_set_super_total_bytes(fs_info->super_copy,
2746                                     orig_super_total_bytes);
2747         btrfs_set_super_num_devices(fs_info->super_copy,
2748                                     orig_super_num_devices);
2749         mutex_unlock(&fs_info->chunk_mutex);
2750         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2751 error_trans:
2752         if (seeding_dev)
2753                 sb->s_flags |= SB_RDONLY;
2754         if (trans)
2755                 btrfs_end_transaction(trans);
2756 error_free_device:
2757         btrfs_free_device(device);
2758 error:
2759         blkdev_put(bdev, FMODE_EXCL);
2760         if (seeding_dev && !unlocked) {
2761                 mutex_unlock(&uuid_mutex);
2762                 up_write(&sb->s_umount);
2763         }
2764         return ret;
2765 }
2766
2767 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2768                                         struct btrfs_device *device)
2769 {
2770         int ret;
2771         struct btrfs_path *path;
2772         struct btrfs_root *root = device->fs_info->chunk_root;
2773         struct btrfs_dev_item *dev_item;
2774         struct extent_buffer *leaf;
2775         struct btrfs_key key;
2776
2777         path = btrfs_alloc_path();
2778         if (!path)
2779                 return -ENOMEM;
2780
2781         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2782         key.type = BTRFS_DEV_ITEM_KEY;
2783         key.offset = device->devid;
2784
2785         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2786         if (ret < 0)
2787                 goto out;
2788
2789         if (ret > 0) {
2790                 ret = -ENOENT;
2791                 goto out;
2792         }
2793
2794         leaf = path->nodes[0];
2795         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2796
2797         btrfs_set_device_id(leaf, dev_item, device->devid);
2798         btrfs_set_device_type(leaf, dev_item, device->type);
2799         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2800         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2801         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2802         btrfs_set_device_total_bytes(leaf, dev_item,
2803                                      btrfs_device_get_disk_total_bytes(device));
2804         btrfs_set_device_bytes_used(leaf, dev_item,
2805                                     btrfs_device_get_bytes_used(device));
2806         btrfs_mark_buffer_dirty(leaf);
2807
2808 out:
2809         btrfs_free_path(path);
2810         return ret;
2811 }
2812
2813 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2814                       struct btrfs_device *device, u64 new_size)
2815 {
2816         struct btrfs_fs_info *fs_info = device->fs_info;
2817         struct btrfs_super_block *super_copy = fs_info->super_copy;
2818         u64 old_total;
2819         u64 diff;
2820
2821         if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2822                 return -EACCES;
2823
2824         new_size = round_down(new_size, fs_info->sectorsize);
2825
2826         mutex_lock(&fs_info->chunk_mutex);
2827         old_total = btrfs_super_total_bytes(super_copy);
2828         diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2829
2830         if (new_size <= device->total_bytes ||
2831             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2832                 mutex_unlock(&fs_info->chunk_mutex);
2833                 return -EINVAL;
2834         }
2835
2836         btrfs_set_super_total_bytes(super_copy,
2837                         round_down(old_total + diff, fs_info->sectorsize));
2838         device->fs_devices->total_rw_bytes += diff;
2839
2840         btrfs_device_set_total_bytes(device, new_size);
2841         btrfs_device_set_disk_total_bytes(device, new_size);
2842         btrfs_clear_space_info_full(device->fs_info);
2843         if (list_empty(&device->post_commit_list))
2844                 list_add_tail(&device->post_commit_list,
2845                               &trans->transaction->dev_update_list);
2846         mutex_unlock(&fs_info->chunk_mutex);
2847
2848         return btrfs_update_device(trans, device);
2849 }
2850
2851 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2852 {
2853         struct btrfs_fs_info *fs_info = trans->fs_info;
2854         struct btrfs_root *root = fs_info->chunk_root;
2855         int ret;
2856         struct btrfs_path *path;
2857         struct btrfs_key key;
2858
2859         path = btrfs_alloc_path();
2860         if (!path)
2861                 return -ENOMEM;
2862
2863         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2864         key.offset = chunk_offset;
2865         key.type = BTRFS_CHUNK_ITEM_KEY;
2866
2867         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2868         if (ret < 0)
2869                 goto out;
2870         else if (ret > 0) { /* Logic error or corruption */
2871                 btrfs_handle_fs_error(fs_info, -ENOENT,
2872                                       "Failed lookup while freeing chunk.");
2873                 ret = -ENOENT;
2874                 goto out;
2875         }
2876
2877         ret = btrfs_del_item(trans, root, path);
2878         if (ret < 0)
2879                 btrfs_handle_fs_error(fs_info, ret,
2880                                       "Failed to delete chunk item.");
2881 out:
2882         btrfs_free_path(path);
2883         return ret;
2884 }
2885
2886 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2887 {
2888         struct btrfs_super_block *super_copy = fs_info->super_copy;
2889         struct btrfs_disk_key *disk_key;
2890         struct btrfs_chunk *chunk;
2891         u8 *ptr;
2892         int ret = 0;
2893         u32 num_stripes;
2894         u32 array_size;
2895         u32 len = 0;
2896         u32 cur;
2897         struct btrfs_key key;
2898
2899         mutex_lock(&fs_info->chunk_mutex);
2900         array_size = btrfs_super_sys_array_size(super_copy);
2901
2902         ptr = super_copy->sys_chunk_array;
2903         cur = 0;
2904
2905         while (cur < array_size) {
2906                 disk_key = (struct btrfs_disk_key *)ptr;
2907                 btrfs_disk_key_to_cpu(&key, disk_key);
2908
2909                 len = sizeof(*disk_key);
2910
2911                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2912                         chunk = (struct btrfs_chunk *)(ptr + len);
2913                         num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2914                         len += btrfs_chunk_item_size(num_stripes);
2915                 } else {
2916                         ret = -EIO;
2917                         break;
2918                 }
2919                 if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2920                     key.offset == chunk_offset) {
2921                         memmove(ptr, ptr + len, array_size - (cur + len));
2922                         array_size -= len;
2923                         btrfs_set_super_sys_array_size(super_copy, array_size);
2924                 } else {
2925                         ptr += len;
2926                         cur += len;
2927                 }
2928         }
2929         mutex_unlock(&fs_info->chunk_mutex);
2930         return ret;
2931 }
2932
2933 /*
2934  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2935  * @logical: Logical block offset in bytes.
2936  * @length: Length of extent in bytes.
2937  *
2938  * Return: Chunk mapping or ERR_PTR.
2939  */
2940 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
2941                                        u64 logical, u64 length)
2942 {
2943         struct extent_map_tree *em_tree;
2944         struct extent_map *em;
2945
2946         em_tree = &fs_info->mapping_tree.map_tree;
2947         read_lock(&em_tree->lock);
2948         em = lookup_extent_mapping(em_tree, logical, length);
2949         read_unlock(&em_tree->lock);
2950
2951         if (!em) {
2952                 btrfs_crit(fs_info, "unable to find logical %llu length %llu",
2953                            logical, length);
2954                 return ERR_PTR(-EINVAL);
2955         }
2956
2957         if (em->start > logical || em->start + em->len < logical) {
2958                 btrfs_crit(fs_info,
2959                            "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
2960                            logical, length, em->start, em->start + em->len);
2961                 free_extent_map(em);
2962                 return ERR_PTR(-EINVAL);
2963         }
2964
2965         /* callers are responsible for dropping em's ref. */
2966         return em;
2967 }
2968
2969 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2970 {
2971         struct btrfs_fs_info *fs_info = trans->fs_info;
2972         struct extent_map *em;
2973         struct map_lookup *map;
2974         u64 dev_extent_len = 0;
2975         int i, ret = 0;
2976         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2977
2978         em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
2979         if (IS_ERR(em)) {
2980                 /*
2981                  * This is a logic error, but we don't want to just rely on the
2982                  * user having built with ASSERT enabled, so if ASSERT doesn't
2983                  * do anything we still error out.
2984                  */
2985                 ASSERT(0);
2986                 return PTR_ERR(em);
2987         }
2988         map = em->map_lookup;
2989         mutex_lock(&fs_info->chunk_mutex);
2990         check_system_chunk(trans, map->type);
2991         mutex_unlock(&fs_info->chunk_mutex);
2992
2993         /*
2994          * Take the device list mutex to prevent races with the final phase of
2995          * a device replace operation that replaces the device object associated
2996          * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
2997          */
2998         mutex_lock(&fs_devices->device_list_mutex);
2999         for (i = 0; i < map->num_stripes; i++) {
3000                 struct btrfs_device *device = map->stripes[i].dev;
3001                 ret = btrfs_free_dev_extent(trans, device,
3002                                             map->stripes[i].physical,
3003                                             &dev_extent_len);
3004                 if (ret) {
3005                         mutex_unlock(&fs_devices->device_list_mutex);
3006                         btrfs_abort_transaction(trans, ret);
3007                         goto out;
3008                 }
3009
3010                 if (device->bytes_used > 0) {
3011                         mutex_lock(&fs_info->chunk_mutex);
3012                         btrfs_device_set_bytes_used(device,
3013                                         device->bytes_used - dev_extent_len);
3014                         atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
3015                         btrfs_clear_space_info_full(fs_info);
3016                         mutex_unlock(&fs_info->chunk_mutex);
3017                 }
3018
3019                 ret = btrfs_update_device(trans, device);
3020                 if (ret) {
3021                         mutex_unlock(&fs_devices->device_list_mutex);
3022                         btrfs_abort_transaction(trans, ret);
3023                         goto out;
3024                 }
3025         }
3026         mutex_unlock(&fs_devices->device_list_mutex);
3027
3028         ret = btrfs_free_chunk(trans, chunk_offset);
3029         if (ret) {
3030                 btrfs_abort_transaction(trans, ret);
3031                 goto out;
3032         }
3033
3034         trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
3035
3036         if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3037                 ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3038                 if (ret) {
3039                         btrfs_abort_transaction(trans, ret);
3040                         goto out;
3041                 }
3042         }
3043
3044         ret = btrfs_remove_block_group(trans, chunk_offset, em);
3045         if (ret) {
3046                 btrfs_abort_transaction(trans, ret);
3047                 goto out;
3048         }
3049
3050 out:
3051         /* once for us */
3052         free_extent_map(em);
3053         return ret;
3054 }
3055
3056 static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3057 {
3058         struct btrfs_root *root = fs_info->chunk_root;
3059         struct btrfs_trans_handle *trans;
3060         int ret;
3061
3062         /*
3063          * Prevent races with automatic removal of unused block groups.
3064          * After we relocate and before we remove the chunk with offset
3065          * chunk_offset, automatic removal of the block group can kick in,
3066          * resulting in a failure when calling btrfs_remove_chunk() below.
3067          *
3068          * Make sure to acquire this mutex before doing a tree search (dev
3069          * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3070          * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3071          * we release the path used to search the chunk/dev tree and before
3072          * the current task acquires this mutex and calls us.
3073          */
3074         lockdep_assert_held(&fs_info->delete_unused_bgs_mutex);
3075
3076         ret = btrfs_can_relocate(fs_info, chunk_offset);
3077         if (ret)
3078                 return -ENOSPC;
3079
3080         /* step one, relocate all the extents inside this chunk */
3081         btrfs_scrub_pause(fs_info);
3082         ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3083         btrfs_scrub_continue(fs_info);
3084         if (ret)
3085                 return ret;
3086
3087         /*
3088          * We add the kobjects here (and after forcing data chunk creation)
3089          * since relocation is the only place we'll create chunks of a new
3090          * type at runtime.  The only place where we'll remove the last
3091          * chunk of a type is the call immediately below this one.  Even
3092          * so, we're protected against races with the cleaner thread since
3093          * we're covered by the delete_unused_bgs_mutex.
3094          */
3095         btrfs_add_raid_kobjects(fs_info);
3096
3097         trans = btrfs_start_trans_remove_block_group(root->fs_info,
3098                                                      chunk_offset);
3099         if (IS_ERR(trans)) {
3100                 ret = PTR_ERR(trans);
3101                 btrfs_handle_fs_error(root->fs_info, ret, NULL);
3102                 return ret;
3103         }
3104
3105         /*
3106          * step two, delete the device extents and the
3107          * chunk tree entries
3108          */
3109         ret = btrfs_remove_chunk(trans, chunk_offset);
3110         btrfs_end_transaction(trans);
3111         return ret;
3112 }
3113
3114 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3115 {
3116         struct btrfs_root *chunk_root = fs_info->chunk_root;
3117         struct btrfs_path *path;
3118         struct extent_buffer *leaf;
3119         struct btrfs_chunk *chunk;
3120         struct btrfs_key key;
3121         struct btrfs_key found_key;
3122         u64 chunk_type;
3123         bool retried = false;
3124         int failed = 0;
3125         int ret;
3126
3127         path = btrfs_alloc_path();
3128         if (!path)
3129                 return -ENOMEM;
3130
3131 again:
3132         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3133         key.offset = (u64)-1;
3134         key.type = BTRFS_CHUNK_ITEM_KEY;
3135
3136         while (1) {
3137                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
3138                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3139                 if (ret < 0) {
3140                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3141                         goto error;
3142                 }
3143                 BUG_ON(ret == 0); /* Corruption */
3144
3145                 ret = btrfs_previous_item(chunk_root, path, key.objectid,
3146                                           key.type);
3147                 if (ret)
3148                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3149                 if (ret < 0)
3150                         goto error;
3151                 if (ret > 0)
3152                         break;
3153
3154                 leaf = path->nodes[0];
3155                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3156
3157                 chunk = btrfs_item_ptr(leaf, path->slots[0],
3158                                        struct btrfs_chunk);
3159                 chunk_type = btrfs_chunk_type(leaf, chunk);
3160                 btrfs_release_path(path);
3161
3162                 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3163                         ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3164                         if (ret == -ENOSPC)
3165                                 failed++;
3166                         else
3167                                 BUG_ON(ret);
3168                 }
3169                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3170
3171                 if (found_key.offset == 0)
3172                         break;
3173                 key.offset = found_key.offset - 1;
3174         }
3175         ret = 0;
3176         if (failed && !retried) {
3177                 failed = 0;
3178                 retried = true;
3179                 goto again;
3180         } else if (WARN_ON(failed && retried)) {
3181                 ret = -ENOSPC;
3182         }
3183 error:
3184         btrfs_free_path(path);
3185         return ret;
3186 }
3187
3188 /*
3189  * return 1 : allocate a data chunk successfully,
3190  * return <0: errors during allocating a data chunk,
3191  * return 0 : no need to allocate a data chunk.
3192  */
3193 static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3194                                       u64 chunk_offset)
3195 {
3196         struct btrfs_block_group_cache *cache;
3197         u64 bytes_used;
3198         u64 chunk_type;
3199
3200         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3201         ASSERT(cache);
3202         chunk_type = cache->flags;
3203         btrfs_put_block_group(cache);
3204
3205         if (chunk_type & BTRFS_BLOCK_GROUP_DATA) {
3206                 spin_lock(&fs_info->data_sinfo->lock);
3207                 bytes_used = fs_info->data_sinfo->bytes_used;
3208                 spin_unlock(&fs_info->data_sinfo->lock);
3209
3210                 if (!bytes_used) {
3211                         struct btrfs_trans_handle *trans;
3212                         int ret;
3213
3214                         trans = btrfs_join_transaction(fs_info->tree_root);
3215                         if (IS_ERR(trans))
3216                                 return PTR_ERR(trans);
3217
3218                         ret = btrfs_force_chunk_alloc(trans,