btrfs: move btrfs_init_dev_replace_tgtdev to dev-replace.c and make static
[sfrench/cifs-2.6.git] / fs / btrfs / dev-replace.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) STRATO AG 2012.  All rights reserved.
4  */
5
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/random.h>
12 #include <linux/iocontext.h>
13 #include <linux/capability.h>
14 #include <linux/kthread.h>
15 #include <linux/math64.h>
16 #include <asm/div64.h>
17 #include "ctree.h"
18 #include "extent_map.h"
19 #include "disk-io.h"
20 #include "transaction.h"
21 #include "print-tree.h"
22 #include "volumes.h"
23 #include "async-thread.h"
24 #include "check-integrity.h"
25 #include "rcu-string.h"
26 #include "dev-replace.h"
27 #include "sysfs.h"
28
29 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
30                                        int scrub_ret);
31 static void btrfs_dev_replace_update_device_in_mapping_tree(
32                                                 struct btrfs_fs_info *fs_info,
33                                                 struct btrfs_device *srcdev,
34                                                 struct btrfs_device *tgtdev);
35 static int btrfs_dev_replace_kthread(void *data);
36
37 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
38 {
39         struct btrfs_key key;
40         struct btrfs_root *dev_root = fs_info->dev_root;
41         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
42         struct extent_buffer *eb;
43         int slot;
44         int ret = 0;
45         struct btrfs_path *path = NULL;
46         int item_size;
47         struct btrfs_dev_replace_item *ptr;
48         u64 src_devid;
49
50         path = btrfs_alloc_path();
51         if (!path) {
52                 ret = -ENOMEM;
53                 goto out;
54         }
55
56         key.objectid = 0;
57         key.type = BTRFS_DEV_REPLACE_KEY;
58         key.offset = 0;
59         ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
60         if (ret) {
61 no_valid_dev_replace_entry_found:
62                 ret = 0;
63                 dev_replace->replace_state =
64                         BTRFS_DEV_REPLACE_ITEM_STATE_NEVER_STARTED;
65                 dev_replace->cont_reading_from_srcdev_mode =
66                     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
67                 dev_replace->replace_state = 0;
68                 dev_replace->time_started = 0;
69                 dev_replace->time_stopped = 0;
70                 atomic64_set(&dev_replace->num_write_errors, 0);
71                 atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
72                 dev_replace->cursor_left = 0;
73                 dev_replace->committed_cursor_left = 0;
74                 dev_replace->cursor_left_last_write_of_item = 0;
75                 dev_replace->cursor_right = 0;
76                 dev_replace->srcdev = NULL;
77                 dev_replace->tgtdev = NULL;
78                 dev_replace->is_valid = 0;
79                 dev_replace->item_needs_writeback = 0;
80                 goto out;
81         }
82         slot = path->slots[0];
83         eb = path->nodes[0];
84         item_size = btrfs_item_size_nr(eb, slot);
85         ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);
86
87         if (item_size != sizeof(struct btrfs_dev_replace_item)) {
88                 btrfs_warn(fs_info,
89                         "dev_replace entry found has unexpected size, ignore entry");
90                 goto no_valid_dev_replace_entry_found;
91         }
92
93         src_devid = btrfs_dev_replace_src_devid(eb, ptr);
94         dev_replace->cont_reading_from_srcdev_mode =
95                 btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
96         dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
97         dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
98         dev_replace->time_stopped =
99                 btrfs_dev_replace_time_stopped(eb, ptr);
100         atomic64_set(&dev_replace->num_write_errors,
101                      btrfs_dev_replace_num_write_errors(eb, ptr));
102         atomic64_set(&dev_replace->num_uncorrectable_read_errors,
103                      btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
104         dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
105         dev_replace->committed_cursor_left = dev_replace->cursor_left;
106         dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
107         dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
108         dev_replace->is_valid = 1;
109
110         dev_replace->item_needs_writeback = 0;
111         switch (dev_replace->replace_state) {
112         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
113         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
114         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
115                 dev_replace->srcdev = NULL;
116                 dev_replace->tgtdev = NULL;
117                 break;
118         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
119         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
120                 dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,
121                                                         NULL, NULL);
122                 dev_replace->tgtdev = btrfs_find_device(fs_info,
123                                                         BTRFS_DEV_REPLACE_DEVID,
124                                                         NULL, NULL);
125                 /*
126                  * allow 'btrfs dev replace_cancel' if src/tgt device is
127                  * missing
128                  */
129                 if (!dev_replace->srcdev &&
130                     !btrfs_test_opt(fs_info, DEGRADED)) {
131                         ret = -EIO;
132                         btrfs_warn(fs_info,
133                            "cannot mount because device replace operation is ongoing and");
134                         btrfs_warn(fs_info,
135                            "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
136                            src_devid);
137                 }
138                 if (!dev_replace->tgtdev &&
139                     !btrfs_test_opt(fs_info, DEGRADED)) {
140                         ret = -EIO;
141                         btrfs_warn(fs_info,
142                            "cannot mount because device replace operation is ongoing and");
143                         btrfs_warn(fs_info,
144                            "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
145                                 BTRFS_DEV_REPLACE_DEVID);
146                 }
147                 if (dev_replace->tgtdev) {
148                         if (dev_replace->srcdev) {
149                                 dev_replace->tgtdev->total_bytes =
150                                         dev_replace->srcdev->total_bytes;
151                                 dev_replace->tgtdev->disk_total_bytes =
152                                         dev_replace->srcdev->disk_total_bytes;
153                                 dev_replace->tgtdev->commit_total_bytes =
154                                         dev_replace->srcdev->commit_total_bytes;
155                                 dev_replace->tgtdev->bytes_used =
156                                         dev_replace->srcdev->bytes_used;
157                                 dev_replace->tgtdev->commit_bytes_used =
158                                         dev_replace->srcdev->commit_bytes_used;
159                         }
160                         set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
161                                 &dev_replace->tgtdev->dev_state);
162
163                         WARN_ON(fs_info->fs_devices->rw_devices == 0);
164                         dev_replace->tgtdev->io_width = fs_info->sectorsize;
165                         dev_replace->tgtdev->io_align = fs_info->sectorsize;
166                         dev_replace->tgtdev->sector_size = fs_info->sectorsize;
167                         dev_replace->tgtdev->fs_info = fs_info;
168                         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
169                                 &dev_replace->tgtdev->dev_state);
170                 }
171                 break;
172         }
173
174 out:
175         btrfs_free_path(path);
176         return ret;
177 }
178
179 /*
180  * Initialize a new device for device replace target from a given source dev
181  * and path.
182  *
183  * Return 0 and new device in @device_out, otherwise return < 0
184  */
185 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
186                                   const char *device_path,
187                                   struct btrfs_device *srcdev,
188                                   struct btrfs_device **device_out)
189 {
190         struct btrfs_device *device;
191         struct block_device *bdev;
192         struct list_head *devices;
193         struct rcu_string *name;
194         u64 devid = BTRFS_DEV_REPLACE_DEVID;
195         int ret = 0;
196
197         *device_out = NULL;
198         if (fs_info->fs_devices->seeding) {
199                 btrfs_err(fs_info, "the filesystem is a seed filesystem!");
200                 return -EINVAL;
201         }
202
203         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
204                                   fs_info->bdev_holder);
205         if (IS_ERR(bdev)) {
206                 btrfs_err(fs_info, "target device %s is invalid!", device_path);
207                 return PTR_ERR(bdev);
208         }
209
210         filemap_write_and_wait(bdev->bd_inode->i_mapping);
211
212         devices = &fs_info->fs_devices->devices;
213         list_for_each_entry(device, devices, dev_list) {
214                 if (device->bdev == bdev) {
215                         btrfs_err(fs_info,
216                                   "target device is in the filesystem!");
217                         ret = -EEXIST;
218                         goto error;
219                 }
220         }
221
222
223         if (i_size_read(bdev->bd_inode) <
224             btrfs_device_get_total_bytes(srcdev)) {
225                 btrfs_err(fs_info,
226                           "target device is smaller than source device!");
227                 ret = -EINVAL;
228                 goto error;
229         }
230
231
232         device = btrfs_alloc_device(NULL, &devid, NULL);
233         if (IS_ERR(device)) {
234                 ret = PTR_ERR(device);
235                 goto error;
236         }
237
238         name = rcu_string_strdup(device_path, GFP_KERNEL);
239         if (!name) {
240                 btrfs_free_device(device);
241                 ret = -ENOMEM;
242                 goto error;
243         }
244         rcu_assign_pointer(device->name, name);
245
246         mutex_lock(&fs_info->fs_devices->device_list_mutex);
247         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
248         device->generation = 0;
249         device->io_width = fs_info->sectorsize;
250         device->io_align = fs_info->sectorsize;
251         device->sector_size = fs_info->sectorsize;
252         device->total_bytes = btrfs_device_get_total_bytes(srcdev);
253         device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
254         device->bytes_used = btrfs_device_get_bytes_used(srcdev);
255         device->commit_total_bytes = srcdev->commit_total_bytes;
256         device->commit_bytes_used = device->bytes_used;
257         device->fs_info = fs_info;
258         device->bdev = bdev;
259         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
260         set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
261         device->mode = FMODE_EXCL;
262         device->dev_stats_valid = 1;
263         set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
264         device->fs_devices = fs_info->fs_devices;
265         list_add(&device->dev_list, &fs_info->fs_devices->devices);
266         fs_info->fs_devices->num_devices++;
267         fs_info->fs_devices->open_devices++;
268         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
269
270         *device_out = device;
271         return 0;
272
273 error:
274         blkdev_put(bdev, FMODE_EXCL);
275         return ret;
276 }
277
278 /*
279  * called from commit_transaction. Writes changed device replace state to
280  * disk.
281  */
282 int btrfs_run_dev_replace(struct btrfs_trans_handle *trans,
283                           struct btrfs_fs_info *fs_info)
284 {
285         int ret;
286         struct btrfs_root *dev_root = fs_info->dev_root;
287         struct btrfs_path *path;
288         struct btrfs_key key;
289         struct extent_buffer *eb;
290         struct btrfs_dev_replace_item *ptr;
291         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
292
293         btrfs_dev_replace_read_lock(dev_replace);
294         if (!dev_replace->is_valid ||
295             !dev_replace->item_needs_writeback) {
296                 btrfs_dev_replace_read_unlock(dev_replace);
297                 return 0;
298         }
299         btrfs_dev_replace_read_unlock(dev_replace);
300
301         key.objectid = 0;
302         key.type = BTRFS_DEV_REPLACE_KEY;
303         key.offset = 0;
304
305         path = btrfs_alloc_path();
306         if (!path) {
307                 ret = -ENOMEM;
308                 goto out;
309         }
310         ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
311         if (ret < 0) {
312                 btrfs_warn(fs_info,
313                            "error %d while searching for dev_replace item!",
314                            ret);
315                 goto out;
316         }
317
318         if (ret == 0 &&
319             btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
320                 /*
321                  * need to delete old one and insert a new one.
322                  * Since no attempt is made to recover any old state, if the
323                  * dev_replace state is 'running', the data on the target
324                  * drive is lost.
325                  * It would be possible to recover the state: just make sure
326                  * that the beginning of the item is never changed and always
327                  * contains all the essential information. Then read this
328                  * minimal set of information and use it as a base for the
329                  * new state.
330                  */
331                 ret = btrfs_del_item(trans, dev_root, path);
332                 if (ret != 0) {
333                         btrfs_warn(fs_info,
334                                    "delete too small dev_replace item failed %d!",
335                                    ret);
336                         goto out;
337                 }
338                 ret = 1;
339         }
340
341         if (ret == 1) {
342                 /* need to insert a new item */
343                 btrfs_release_path(path);
344                 ret = btrfs_insert_empty_item(trans, dev_root, path,
345                                               &key, sizeof(*ptr));
346                 if (ret < 0) {
347                         btrfs_warn(fs_info,
348                                    "insert dev_replace item failed %d!", ret);
349                         goto out;
350                 }
351         }
352
353         eb = path->nodes[0];
354         ptr = btrfs_item_ptr(eb, path->slots[0],
355                              struct btrfs_dev_replace_item);
356
357         btrfs_dev_replace_write_lock(dev_replace);
358         if (dev_replace->srcdev)
359                 btrfs_set_dev_replace_src_devid(eb, ptr,
360                         dev_replace->srcdev->devid);
361         else
362                 btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
363         btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
364                 dev_replace->cont_reading_from_srcdev_mode);
365         btrfs_set_dev_replace_replace_state(eb, ptr,
366                 dev_replace->replace_state);
367         btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
368         btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
369         btrfs_set_dev_replace_num_write_errors(eb, ptr,
370                 atomic64_read(&dev_replace->num_write_errors));
371         btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
372                 atomic64_read(&dev_replace->num_uncorrectable_read_errors));
373         dev_replace->cursor_left_last_write_of_item =
374                 dev_replace->cursor_left;
375         btrfs_set_dev_replace_cursor_left(eb, ptr,
376                 dev_replace->cursor_left_last_write_of_item);
377         btrfs_set_dev_replace_cursor_right(eb, ptr,
378                 dev_replace->cursor_right);
379         dev_replace->item_needs_writeback = 0;
380         btrfs_dev_replace_write_unlock(dev_replace);
381
382         btrfs_mark_buffer_dirty(eb);
383
384 out:
385         btrfs_free_path(path);
386
387         return ret;
388 }
389
390 void btrfs_after_dev_replace_commit(struct btrfs_fs_info *fs_info)
391 {
392         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
393
394         dev_replace->committed_cursor_left =
395                 dev_replace->cursor_left_last_write_of_item;
396 }
397
398 static char* btrfs_dev_name(struct btrfs_device *device)
399 {
400         if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
401                 return "<missing disk>";
402         else
403                 return rcu_str_deref(device->name);
404 }
405
406 int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
407                 const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
408                 int read_src)
409 {
410         struct btrfs_root *root = fs_info->dev_root;
411         struct btrfs_trans_handle *trans;
412         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
413         int ret;
414         struct btrfs_device *tgt_device = NULL;
415         struct btrfs_device *src_device = NULL;
416
417         /* the disk copy procedure reuses the scrub code */
418         mutex_lock(&fs_info->volume_mutex);
419         ret = btrfs_find_device_by_devspec(fs_info, srcdevid,
420                                             srcdev_name, &src_device);
421         if (ret) {
422                 mutex_unlock(&fs_info->volume_mutex);
423                 return ret;
424         }
425
426         ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
427                                             src_device, &tgt_device);
428         mutex_unlock(&fs_info->volume_mutex);
429         if (ret)
430                 return ret;
431
432         /*
433          * Here we commit the transaction to make sure commit_total_bytes
434          * of all the devices are updated.
435          */
436         trans = btrfs_attach_transaction(root);
437         if (!IS_ERR(trans)) {
438                 ret = btrfs_commit_transaction(trans);
439                 if (ret)
440                         return ret;
441         } else if (PTR_ERR(trans) != -ENOENT) {
442                 return PTR_ERR(trans);
443         }
444
445         btrfs_dev_replace_write_lock(dev_replace);
446         switch (dev_replace->replace_state) {
447         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
448         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
449         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
450                 break;
451         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
452         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
453                 ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
454                 goto leave;
455         }
456
457         dev_replace->cont_reading_from_srcdev_mode = read_src;
458         WARN_ON(!src_device);
459         dev_replace->srcdev = src_device;
460         dev_replace->tgtdev = tgt_device;
461
462         btrfs_info_in_rcu(fs_info,
463                       "dev_replace from %s (devid %llu) to %s started",
464                       btrfs_dev_name(src_device),
465                       src_device->devid,
466                       rcu_str_deref(tgt_device->name));
467
468         /*
469          * from now on, the writes to the srcdev are all duplicated to
470          * go to the tgtdev as well (refer to btrfs_map_block()).
471          */
472         dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
473         dev_replace->time_started = get_seconds();
474         dev_replace->cursor_left = 0;
475         dev_replace->committed_cursor_left = 0;
476         dev_replace->cursor_left_last_write_of_item = 0;
477         dev_replace->cursor_right = 0;
478         dev_replace->is_valid = 1;
479         dev_replace->item_needs_writeback = 1;
480         atomic64_set(&dev_replace->num_write_errors, 0);
481         atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
482         btrfs_dev_replace_write_unlock(dev_replace);
483
484         ret = btrfs_sysfs_add_device_link(tgt_device->fs_devices, tgt_device);
485         if (ret)
486                 btrfs_err(fs_info, "kobj add dev failed %d", ret);
487
488         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
489
490         /* force writing the updated state information to disk */
491         trans = btrfs_start_transaction(root, 0);
492         if (IS_ERR(trans)) {
493                 ret = PTR_ERR(trans);
494                 btrfs_dev_replace_write_lock(dev_replace);
495                 goto leave;
496         }
497
498         ret = btrfs_commit_transaction(trans);
499         WARN_ON(ret);
500
501         /* the disk copy procedure reuses the scrub code */
502         ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
503                               btrfs_device_get_total_bytes(src_device),
504                               &dev_replace->scrub_progress, 0, 1);
505
506         ret = btrfs_dev_replace_finishing(fs_info, ret);
507         if (ret == -EINPROGRESS) {
508                 ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
509         } else {
510                 WARN_ON(ret);
511         }
512
513         return ret;
514
515 leave:
516         dev_replace->srcdev = NULL;
517         dev_replace->tgtdev = NULL;
518         btrfs_dev_replace_write_unlock(dev_replace);
519         btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);
520         return ret;
521 }
522
523 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
524                             struct btrfs_ioctl_dev_replace_args *args)
525 {
526         int ret;
527
528         switch (args->start.cont_reading_from_srcdev_mode) {
529         case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
530         case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
531                 break;
532         default:
533                 return -EINVAL;
534         }
535
536         if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') ||
537             args->start.tgtdev_name[0] == '\0')
538                 return -EINVAL;
539
540         ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
541                                         args->start.srcdevid,
542                                         args->start.srcdev_name,
543                                         args->start.cont_reading_from_srcdev_mode);
544         args->result = ret;
545         /* don't warn if EINPROGRESS, someone else might be running scrub */
546         if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS)
547                 ret = 0;
548
549         return ret;
550 }
551
552 /*
553  * blocked until all in-flight bios operations are finished.
554  */
555 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
556 {
557         set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
558         wait_event(fs_info->replace_wait, !percpu_counter_sum(
559                    &fs_info->bio_counter));
560 }
561
562 /*
563  * we have removed target device, it is safe to allow new bios request.
564  */
565 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
566 {
567         clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
568         wake_up(&fs_info->replace_wait);
569 }
570
571 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
572                                        int scrub_ret)
573 {
574         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
575         struct btrfs_device *tgt_device;
576         struct btrfs_device *src_device;
577         struct btrfs_root *root = fs_info->tree_root;
578         u8 uuid_tmp[BTRFS_UUID_SIZE];
579         struct btrfs_trans_handle *trans;
580         int ret = 0;
581
582         /* don't allow cancel or unmount to disturb the finishing procedure */
583         mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
584
585         btrfs_dev_replace_read_lock(dev_replace);
586         /* was the operation canceled, or is it finished? */
587         if (dev_replace->replace_state !=
588             BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
589                 btrfs_dev_replace_read_unlock(dev_replace);
590                 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
591                 return 0;
592         }
593
594         tgt_device = dev_replace->tgtdev;
595         src_device = dev_replace->srcdev;
596         btrfs_dev_replace_read_unlock(dev_replace);
597
598         /*
599          * flush all outstanding I/O and inode extent mappings before the
600          * copy operation is declared as being finished
601          */
602         ret = btrfs_start_delalloc_roots(fs_info, 0, -1);
603         if (ret) {
604                 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
605                 return ret;
606         }
607         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
608
609         trans = btrfs_start_transaction(root, 0);
610         if (IS_ERR(trans)) {
611                 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
612                 return PTR_ERR(trans);
613         }
614         ret = btrfs_commit_transaction(trans);
615         WARN_ON(ret);
616
617         mutex_lock(&uuid_mutex);
618         /* keep away write_all_supers() during the finishing procedure */
619         mutex_lock(&fs_info->fs_devices->device_list_mutex);
620         mutex_lock(&fs_info->chunk_mutex);
621         btrfs_dev_replace_write_lock(dev_replace);
622         dev_replace->replace_state =
623                 scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
624                           : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
625         dev_replace->tgtdev = NULL;
626         dev_replace->srcdev = NULL;
627         dev_replace->time_stopped = get_seconds();
628         dev_replace->item_needs_writeback = 1;
629
630         /* replace old device with new one in mapping tree */
631         if (!scrub_ret) {
632                 btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
633                                                                 src_device,
634                                                                 tgt_device);
635         } else {
636                 btrfs_err_in_rcu(fs_info,
637                                  "btrfs_scrub_dev(%s, %llu, %s) failed %d",
638                                  btrfs_dev_name(src_device),
639                                  src_device->devid,
640                                  rcu_str_deref(tgt_device->name), scrub_ret);
641                 btrfs_dev_replace_write_unlock(dev_replace);
642                 mutex_unlock(&fs_info->chunk_mutex);
643                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
644                 mutex_unlock(&uuid_mutex);
645                 btrfs_rm_dev_replace_blocked(fs_info);
646                 if (tgt_device)
647                         btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);
648                 btrfs_rm_dev_replace_unblocked(fs_info);
649                 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
650
651                 return scrub_ret;
652         }
653
654         btrfs_info_in_rcu(fs_info,
655                           "dev_replace from %s (devid %llu) to %s finished",
656                           btrfs_dev_name(src_device),
657                           src_device->devid,
658                           rcu_str_deref(tgt_device->name));
659         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
660         tgt_device->devid = src_device->devid;
661         src_device->devid = BTRFS_DEV_REPLACE_DEVID;
662         memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
663         memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
664         memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
665         btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
666         btrfs_device_set_disk_total_bytes(tgt_device,
667                                           src_device->disk_total_bytes);
668         btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
669         ASSERT(list_empty(&src_device->resized_list));
670         tgt_device->commit_total_bytes = src_device->commit_total_bytes;
671         tgt_device->commit_bytes_used = src_device->bytes_used;
672
673         btrfs_assign_next_active_device(fs_info, src_device, tgt_device);
674
675         list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);
676         fs_info->fs_devices->rw_devices++;
677
678         btrfs_dev_replace_write_unlock(dev_replace);
679
680         btrfs_rm_dev_replace_blocked(fs_info);
681
682         btrfs_rm_dev_replace_remove_srcdev(fs_info, src_device);
683
684         btrfs_rm_dev_replace_unblocked(fs_info);
685
686         /*
687          * this is again a consistent state where no dev_replace procedure
688          * is running, the target device is part of the filesystem, the
689          * source device is not part of the filesystem anymore and its 1st
690          * superblock is scratched out so that it is no longer marked to
691          * belong to this filesystem.
692          */
693         mutex_unlock(&fs_info->chunk_mutex);
694         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
695         mutex_unlock(&uuid_mutex);
696
697         /* replace the sysfs entry */
698         btrfs_sysfs_rm_device_link(fs_info->fs_devices, src_device);
699         btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);
700
701         /* write back the superblocks */
702         trans = btrfs_start_transaction(root, 0);
703         if (!IS_ERR(trans))
704                 btrfs_commit_transaction(trans);
705
706         mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
707
708         return 0;
709 }
710
711 static void btrfs_dev_replace_update_device_in_mapping_tree(
712                                                 struct btrfs_fs_info *fs_info,
713                                                 struct btrfs_device *srcdev,
714                                                 struct btrfs_device *tgtdev)
715 {
716         struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;
717         struct extent_map *em;
718         struct map_lookup *map;
719         u64 start = 0;
720         int i;
721
722         write_lock(&em_tree->lock);
723         do {
724                 em = lookup_extent_mapping(em_tree, start, (u64)-1);
725                 if (!em)
726                         break;
727                 map = em->map_lookup;
728                 for (i = 0; i < map->num_stripes; i++)
729                         if (srcdev == map->stripes[i].dev)
730                                 map->stripes[i].dev = tgtdev;
731                 start = em->start + em->len;
732                 free_extent_map(em);
733         } while (start);
734         write_unlock(&em_tree->lock);
735 }
736
737 /*
738  * Read progress of device replace status according to the state and last
739  * stored position. The value format is the same as for
740  * btrfs_dev_replace::progress_1000
741  */
742 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
743 {
744         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
745         u64 ret = 0;
746
747         switch (dev_replace->replace_state) {
748         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
749         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
750                 ret = 0;
751                 break;
752         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
753                 ret = 1000;
754                 break;
755         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
756         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
757                 ret = div64_u64(dev_replace->cursor_left,
758                                 div_u64(btrfs_device_get_total_bytes(
759                                                 dev_replace->srcdev), 1000));
760                 break;
761         }
762
763         return ret;
764 }
765
766 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
767                               struct btrfs_ioctl_dev_replace_args *args)
768 {
769         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
770
771         btrfs_dev_replace_read_lock(dev_replace);
772         /* even if !dev_replace_is_valid, the values are good enough for
773          * the replace_status ioctl */
774         args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
775         args->status.replace_state = dev_replace->replace_state;
776         args->status.time_started = dev_replace->time_started;
777         args->status.time_stopped = dev_replace->time_stopped;
778         args->status.num_write_errors =
779                 atomic64_read(&dev_replace->num_write_errors);
780         args->status.num_uncorrectable_read_errors =
781                 atomic64_read(&dev_replace->num_uncorrectable_read_errors);
782         args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
783         btrfs_dev_replace_read_unlock(dev_replace);
784 }
785
786 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
787 {
788         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
789         struct btrfs_device *tgt_device = NULL;
790         struct btrfs_device *src_device = NULL;
791         struct btrfs_trans_handle *trans;
792         struct btrfs_root *root = fs_info->tree_root;
793         int result;
794         int ret;
795
796         if (sb_rdonly(fs_info->sb))
797                 return -EROFS;
798
799         mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
800         btrfs_dev_replace_write_lock(dev_replace);
801         switch (dev_replace->replace_state) {
802         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
803         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
804         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
805                 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
806                 btrfs_dev_replace_write_unlock(dev_replace);
807                 goto leave;
808         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
809         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
810                 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
811                 tgt_device = dev_replace->tgtdev;
812                 src_device = dev_replace->srcdev;
813                 dev_replace->tgtdev = NULL;
814                 dev_replace->srcdev = NULL;
815                 break;
816         }
817         dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
818         dev_replace->time_stopped = get_seconds();
819         dev_replace->item_needs_writeback = 1;
820         btrfs_dev_replace_write_unlock(dev_replace);
821         btrfs_scrub_cancel(fs_info);
822
823         trans = btrfs_start_transaction(root, 0);
824         if (IS_ERR(trans)) {
825                 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
826                 return PTR_ERR(trans);
827         }
828         ret = btrfs_commit_transaction(trans);
829         WARN_ON(ret);
830
831         btrfs_info_in_rcu(fs_info,
832                 "dev_replace from %s (devid %llu) to %s canceled",
833                 btrfs_dev_name(src_device), src_device->devid,
834                 btrfs_dev_name(tgt_device));
835
836         if (tgt_device)
837                 btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);
838
839 leave:
840         mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
841         return result;
842 }
843
844 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
845 {
846         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
847
848         mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
849         btrfs_dev_replace_write_lock(dev_replace);
850         switch (dev_replace->replace_state) {
851         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
852         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
853         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
854         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
855                 break;
856         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
857                 dev_replace->replace_state =
858                         BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
859                 dev_replace->time_stopped = get_seconds();
860                 dev_replace->item_needs_writeback = 1;
861                 btrfs_info(fs_info, "suspending dev_replace for unmount");
862                 break;
863         }
864
865         btrfs_dev_replace_write_unlock(dev_replace);
866         mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
867 }
868
869 /* resume dev_replace procedure that was interrupted by unmount */
870 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
871 {
872         struct task_struct *task;
873         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
874
875         btrfs_dev_replace_write_lock(dev_replace);
876         switch (dev_replace->replace_state) {
877         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
878         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
879         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
880                 btrfs_dev_replace_write_unlock(dev_replace);
881                 return 0;
882         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
883                 break;
884         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
885                 dev_replace->replace_state =
886                         BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
887                 break;
888         }
889         if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
890                 btrfs_info(fs_info,
891                            "cannot continue dev_replace, tgtdev is missing");
892                 btrfs_info(fs_info,
893                            "you may cancel the operation after 'mount -o degraded'");
894                 btrfs_dev_replace_write_unlock(dev_replace);
895                 return 0;
896         }
897         btrfs_dev_replace_write_unlock(dev_replace);
898
899         WARN_ON(test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags));
900         task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
901         return PTR_ERR_OR_ZERO(task);
902 }
903
904 static int btrfs_dev_replace_kthread(void *data)
905 {
906         struct btrfs_fs_info *fs_info = data;
907         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
908         u64 progress;
909         int ret;
910
911         progress = btrfs_dev_replace_progress(fs_info);
912         progress = div_u64(progress, 10);
913         btrfs_info_in_rcu(fs_info,
914                 "continuing dev_replace from %s (devid %llu) to target %s @%u%%",
915                 btrfs_dev_name(dev_replace->srcdev),
916                 dev_replace->srcdev->devid,
917                 btrfs_dev_name(dev_replace->tgtdev),
918                 (unsigned int)progress);
919
920         ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
921                               dev_replace->committed_cursor_left,
922                               btrfs_device_get_total_bytes(dev_replace->srcdev),
923                               &dev_replace->scrub_progress, 0, 1);
924         ret = btrfs_dev_replace_finishing(fs_info, ret);
925         WARN_ON(ret);
926
927         clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
928         return 0;
929 }
930
931 int btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
932 {
933         if (!dev_replace->is_valid)
934                 return 0;
935
936         switch (dev_replace->replace_state) {
937         case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
938         case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
939         case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
940                 return 0;
941         case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
942         case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
943                 /*
944                  * return true even if tgtdev is missing (this is
945                  * something that can happen if the dev_replace
946                  * procedure is suspended by an umount and then
947                  * the tgtdev is missing (or "btrfs dev scan") was
948                  * not called and the the filesystem is remounted
949                  * in degraded state. This does not stop the
950                  * dev_replace procedure. It needs to be canceled
951                  * manually if the cancellation is wanted.
952                  */
953                 break;
954         }
955         return 1;
956 }
957
958 void btrfs_dev_replace_read_lock(struct btrfs_dev_replace *dev_replace)
959 {
960         read_lock(&dev_replace->lock);
961         atomic_inc(&dev_replace->read_locks);
962 }
963
964 void btrfs_dev_replace_read_unlock(struct btrfs_dev_replace *dev_replace)
965 {
966         ASSERT(atomic_read(&dev_replace->read_locks) > 0);
967         atomic_dec(&dev_replace->read_locks);
968         read_unlock(&dev_replace->lock);
969 }
970
971 void btrfs_dev_replace_write_lock(struct btrfs_dev_replace *dev_replace)
972 {
973 again:
974         wait_event(dev_replace->read_lock_wq,
975                    atomic_read(&dev_replace->blocking_readers) == 0);
976         write_lock(&dev_replace->lock);
977         if (atomic_read(&dev_replace->blocking_readers)) {
978                 write_unlock(&dev_replace->lock);
979                 goto again;
980         }
981 }
982
983 void btrfs_dev_replace_write_unlock(struct btrfs_dev_replace *dev_replace)
984 {
985         ASSERT(atomic_read(&dev_replace->blocking_readers) == 0);
986         write_unlock(&dev_replace->lock);
987 }
988
989 /* inc blocking cnt and release read lock */
990 void btrfs_dev_replace_set_lock_blocking(
991                                         struct btrfs_dev_replace *dev_replace)
992 {
993         /* only set blocking for read lock */
994         ASSERT(atomic_read(&dev_replace->read_locks) > 0);
995         atomic_inc(&dev_replace->blocking_readers);
996         read_unlock(&dev_replace->lock);
997 }
998
999 /* acquire read lock and dec blocking cnt */
1000 void btrfs_dev_replace_clear_lock_blocking(
1001                                         struct btrfs_dev_replace *dev_replace)
1002 {
1003         /* only set blocking for read lock */
1004         ASSERT(atomic_read(&dev_replace->read_locks) > 0);
1005         ASSERT(atomic_read(&dev_replace->blocking_readers) > 0);
1006         read_lock(&dev_replace->lock);
1007         if (atomic_dec_and_test(&dev_replace->blocking_readers) &&
1008             waitqueue_active(&dev_replace->read_lock_wq))
1009                 wake_up(&dev_replace->read_lock_wq);
1010 }
1011
1012 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)
1013 {
1014         percpu_counter_inc(&fs_info->bio_counter);
1015 }
1016
1017 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1018 {
1019         percpu_counter_sub(&fs_info->bio_counter, amount);
1020
1021         if (waitqueue_active(&fs_info->replace_wait))
1022                 wake_up(&fs_info->replace_wait);
1023 }
1024
1025 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
1026 {
1027         while (1) {
1028                 percpu_counter_inc(&fs_info->bio_counter);
1029                 if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1030                                      &fs_info->fs_state)))
1031                         break;
1032
1033                 btrfs_bio_counter_dec(fs_info);
1034                 wait_event(fs_info->replace_wait,
1035                            !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1036                                      &fs_info->fs_state));
1037         }
1038 }