e08b6bc676e3faa4d93c211b11a59a90c1a62891
[sfrench/cifs-2.6.git] / fs / btrfs / scrub.c
1 /*
2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public
14  * License along with this program; if not, write to the
15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16  * Boston, MA 021110-1307, USA.
17  */
18
19 #include <linux/blkdev.h>
20 #include <linux/ratelimit.h>
21 #include "ctree.h"
22 #include "volumes.h"
23 #include "disk-io.h"
24 #include "ordered-data.h"
25 #include "transaction.h"
26 #include "backref.h"
27 #include "extent_io.h"
28 #include "dev-replace.h"
29 #include "check-integrity.h"
30 #include "rcu-string.h"
31 #include "raid56.h"
32
33 /*
34  * This is only the first step towards a full-features scrub. It reads all
35  * extent and super block and verifies the checksums. In case a bad checksum
36  * is found or the extent cannot be read, good data will be written back if
37  * any can be found.
38  *
39  * Future enhancements:
40  *  - In case an unrepairable extent is encountered, track which files are
41  *    affected and report them
42  *  - track and record media errors, throw out bad devices
43  *  - add a mode to also read unallocated space
44  */
45
46 struct scrub_block;
47 struct scrub_ctx;
48
49 /*
50  * the following three values only influence the performance.
51  * The last one configures the number of parallel and outstanding I/O
52  * operations. The first two values configure an upper limit for the number
53  * of (dynamically allocated) pages that are added to a bio.
54  */
55 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
56 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
57 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
58
59 /*
60  * the following value times PAGE_SIZE needs to be large enough to match the
61  * largest node/leaf/sector size that shall be supported.
62  * Values larger than BTRFS_STRIPE_LEN are not supported.
63  */
64 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
65
66 struct scrub_recover {
67         atomic_t                refs;
68         struct btrfs_bio        *bbio;
69         u64                     map_length;
70 };
71
72 struct scrub_page {
73         struct scrub_block      *sblock;
74         struct page             *page;
75         struct btrfs_device     *dev;
76         struct list_head        list;
77         u64                     flags;  /* extent flags */
78         u64                     generation;
79         u64                     logical;
80         u64                     physical;
81         u64                     physical_for_dev_replace;
82         atomic_t                refs;
83         struct {
84                 unsigned int    mirror_num:8;
85                 unsigned int    have_csum:1;
86                 unsigned int    io_error:1;
87         };
88         u8                      csum[BTRFS_CSUM_SIZE];
89
90         struct scrub_recover    *recover;
91 };
92
93 struct scrub_bio {
94         int                     index;
95         struct scrub_ctx        *sctx;
96         struct btrfs_device     *dev;
97         struct bio              *bio;
98         int                     err;
99         u64                     logical;
100         u64                     physical;
101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
102         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
103 #else
104         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
105 #endif
106         int                     page_count;
107         int                     next_free;
108         struct btrfs_work       work;
109 };
110
111 struct scrub_block {
112         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
113         int                     page_count;
114         atomic_t                outstanding_pages;
115         atomic_t                refs; /* free mem on transition to zero */
116         struct scrub_ctx        *sctx;
117         struct scrub_parity     *sparity;
118         struct {
119                 unsigned int    header_error:1;
120                 unsigned int    checksum_error:1;
121                 unsigned int    no_io_error_seen:1;
122                 unsigned int    generation_error:1; /* also sets header_error */
123
124                 /* The following is for the data used to check parity */
125                 /* It is for the data with checksum */
126                 unsigned int    data_corrected:1;
127         };
128         struct btrfs_work       work;
129 };
130
131 /* Used for the chunks with parity stripe such RAID5/6 */
132 struct scrub_parity {
133         struct scrub_ctx        *sctx;
134
135         struct btrfs_device     *scrub_dev;
136
137         u64                     logic_start;
138
139         u64                     logic_end;
140
141         int                     nsectors;
142
143         int                     stripe_len;
144
145         atomic_t                refs;
146
147         struct list_head        spages;
148
149         /* Work of parity check and repair */
150         struct btrfs_work       work;
151
152         /* Mark the parity blocks which have data */
153         unsigned long           *dbitmap;
154
155         /*
156          * Mark the parity blocks which have data, but errors happen when
157          * read data or check data
158          */
159         unsigned long           *ebitmap;
160
161         unsigned long           bitmap[0];
162 };
163
164 struct scrub_wr_ctx {
165         struct scrub_bio *wr_curr_bio;
166         struct btrfs_device *tgtdev;
167         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
168         atomic_t flush_all_writes;
169         struct mutex wr_lock;
170 };
171
172 struct scrub_ctx {
173         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
174         struct btrfs_root       *dev_root;
175         int                     first_free;
176         int                     curr;
177         atomic_t                bios_in_flight;
178         atomic_t                workers_pending;
179         spinlock_t              list_lock;
180         wait_queue_head_t       list_wait;
181         u16                     csum_size;
182         struct list_head        csum_list;
183         atomic_t                cancel_req;
184         int                     readonly;
185         int                     pages_per_rd_bio;
186         u32                     sectorsize;
187         u32                     nodesize;
188
189         int                     is_dev_replace;
190         struct scrub_wr_ctx     wr_ctx;
191
192         /*
193          * statistics
194          */
195         struct btrfs_scrub_progress stat;
196         spinlock_t              stat_lock;
197
198         /*
199          * Use a ref counter to avoid use-after-free issues. Scrub workers
200          * decrement bios_in_flight and workers_pending and then do a wakeup
201          * on the list_wait wait queue. We must ensure the main scrub task
202          * doesn't free the scrub context before or while the workers are
203          * doing the wakeup() call.
204          */
205         atomic_t                refs;
206 };
207
208 struct scrub_fixup_nodatasum {
209         struct scrub_ctx        *sctx;
210         struct btrfs_device     *dev;
211         u64                     logical;
212         struct btrfs_root       *root;
213         struct btrfs_work       work;
214         int                     mirror_num;
215 };
216
217 struct scrub_nocow_inode {
218         u64                     inum;
219         u64                     offset;
220         u64                     root;
221         struct list_head        list;
222 };
223
224 struct scrub_copy_nocow_ctx {
225         struct scrub_ctx        *sctx;
226         u64                     logical;
227         u64                     len;
228         int                     mirror_num;
229         u64                     physical_for_dev_replace;
230         struct list_head        inodes;
231         struct btrfs_work       work;
232 };
233
234 struct scrub_warning {
235         struct btrfs_path       *path;
236         u64                     extent_item_size;
237         const char              *errstr;
238         sector_t                sector;
239         u64                     logical;
240         struct btrfs_device     *dev;
241 };
242
243 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
244 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
245 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
246 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
247 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
248 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
249                                      struct scrub_block *sblocks_for_recheck);
250 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
251                                 struct scrub_block *sblock,
252                                 int retry_failed_mirror);
253 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
254 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
255                                              struct scrub_block *sblock_good);
256 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
257                                             struct scrub_block *sblock_good,
258                                             int page_num, int force_write);
259 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
260 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
261                                            int page_num);
262 static int scrub_checksum_data(struct scrub_block *sblock);
263 static int scrub_checksum_tree_block(struct scrub_block *sblock);
264 static int scrub_checksum_super(struct scrub_block *sblock);
265 static void scrub_block_get(struct scrub_block *sblock);
266 static void scrub_block_put(struct scrub_block *sblock);
267 static void scrub_page_get(struct scrub_page *spage);
268 static void scrub_page_put(struct scrub_page *spage);
269 static void scrub_parity_get(struct scrub_parity *sparity);
270 static void scrub_parity_put(struct scrub_parity *sparity);
271 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
272                                     struct scrub_page *spage);
273 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
274                        u64 physical, struct btrfs_device *dev, u64 flags,
275                        u64 gen, int mirror_num, u8 *csum, int force,
276                        u64 physical_for_dev_replace);
277 static void scrub_bio_end_io(struct bio *bio);
278 static void scrub_bio_end_io_worker(struct btrfs_work *work);
279 static void scrub_block_complete(struct scrub_block *sblock);
280 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
281                                u64 extent_logical, u64 extent_len,
282                                u64 *extent_physical,
283                                struct btrfs_device **extent_dev,
284                                int *extent_mirror_num);
285 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
286                               struct scrub_wr_ctx *wr_ctx,
287                               struct btrfs_fs_info *fs_info,
288                               struct btrfs_device *dev,
289                               int is_dev_replace);
290 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
291 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
292                                     struct scrub_page *spage);
293 static void scrub_wr_submit(struct scrub_ctx *sctx);
294 static void scrub_wr_bio_end_io(struct bio *bio);
295 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
296 static int write_page_nocow(struct scrub_ctx *sctx,
297                             u64 physical_for_dev_replace, struct page *page);
298 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
299                                       struct scrub_copy_nocow_ctx *ctx);
300 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
301                             int mirror_num, u64 physical_for_dev_replace);
302 static void copy_nocow_pages_worker(struct btrfs_work *work);
303 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
304 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
305 static void scrub_put_ctx(struct scrub_ctx *sctx);
306
307
308 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
309 {
310         atomic_inc(&sctx->refs);
311         atomic_inc(&sctx->bios_in_flight);
312 }
313
314 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
315 {
316         atomic_dec(&sctx->bios_in_flight);
317         wake_up(&sctx->list_wait);
318         scrub_put_ctx(sctx);
319 }
320
321 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
322 {
323         while (atomic_read(&fs_info->scrub_pause_req)) {
324                 mutex_unlock(&fs_info->scrub_lock);
325                 wait_event(fs_info->scrub_pause_wait,
326                    atomic_read(&fs_info->scrub_pause_req) == 0);
327                 mutex_lock(&fs_info->scrub_lock);
328         }
329 }
330
331 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
332 {
333         atomic_inc(&fs_info->scrubs_paused);
334         wake_up(&fs_info->scrub_pause_wait);
335 }
336
337 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
338 {
339         mutex_lock(&fs_info->scrub_lock);
340         __scrub_blocked_if_needed(fs_info);
341         atomic_dec(&fs_info->scrubs_paused);
342         mutex_unlock(&fs_info->scrub_lock);
343
344         wake_up(&fs_info->scrub_pause_wait);
345 }
346
347 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
348 {
349         scrub_pause_on(fs_info);
350         scrub_pause_off(fs_info);
351 }
352
353 /*
354  * used for workers that require transaction commits (i.e., for the
355  * NOCOW case)
356  */
357 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
358 {
359         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
360
361         atomic_inc(&sctx->refs);
362         /*
363          * increment scrubs_running to prevent cancel requests from
364          * completing as long as a worker is running. we must also
365          * increment scrubs_paused to prevent deadlocking on pause
366          * requests used for transactions commits (as the worker uses a
367          * transaction context). it is safe to regard the worker
368          * as paused for all matters practical. effectively, we only
369          * avoid cancellation requests from completing.
370          */
371         mutex_lock(&fs_info->scrub_lock);
372         atomic_inc(&fs_info->scrubs_running);
373         atomic_inc(&fs_info->scrubs_paused);
374         mutex_unlock(&fs_info->scrub_lock);
375
376         /*
377          * check if @scrubs_running=@scrubs_paused condition
378          * inside wait_event() is not an atomic operation.
379          * which means we may inc/dec @scrub_running/paused
380          * at any time. Let's wake up @scrub_pause_wait as
381          * much as we can to let commit transaction blocked less.
382          */
383         wake_up(&fs_info->scrub_pause_wait);
384
385         atomic_inc(&sctx->workers_pending);
386 }
387
388 /* used for workers that require transaction commits */
389 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
390 {
391         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
392
393         /*
394          * see scrub_pending_trans_workers_inc() why we're pretending
395          * to be paused in the scrub counters
396          */
397         mutex_lock(&fs_info->scrub_lock);
398         atomic_dec(&fs_info->scrubs_running);
399         atomic_dec(&fs_info->scrubs_paused);
400         mutex_unlock(&fs_info->scrub_lock);
401         atomic_dec(&sctx->workers_pending);
402         wake_up(&fs_info->scrub_pause_wait);
403         wake_up(&sctx->list_wait);
404         scrub_put_ctx(sctx);
405 }
406
407 static void scrub_free_csums(struct scrub_ctx *sctx)
408 {
409         while (!list_empty(&sctx->csum_list)) {
410                 struct btrfs_ordered_sum *sum;
411                 sum = list_first_entry(&sctx->csum_list,
412                                        struct btrfs_ordered_sum, list);
413                 list_del(&sum->list);
414                 kfree(sum);
415         }
416 }
417
418 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
419 {
420         int i;
421
422         if (!sctx)
423                 return;
424
425         scrub_free_wr_ctx(&sctx->wr_ctx);
426
427         /* this can happen when scrub is cancelled */
428         if (sctx->curr != -1) {
429                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
430
431                 for (i = 0; i < sbio->page_count; i++) {
432                         WARN_ON(!sbio->pagev[i]->page);
433                         scrub_block_put(sbio->pagev[i]->sblock);
434                 }
435                 bio_put(sbio->bio);
436         }
437
438         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
439                 struct scrub_bio *sbio = sctx->bios[i];
440
441                 if (!sbio)
442                         break;
443                 kfree(sbio);
444         }
445
446         scrub_free_csums(sctx);
447         kfree(sctx);
448 }
449
450 static void scrub_put_ctx(struct scrub_ctx *sctx)
451 {
452         if (atomic_dec_and_test(&sctx->refs))
453                 scrub_free_ctx(sctx);
454 }
455
456 static noinline_for_stack
457 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
458 {
459         struct scrub_ctx *sctx;
460         int             i;
461         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
462         int ret;
463
464         sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
465         if (!sctx)
466                 goto nomem;
467         atomic_set(&sctx->refs, 1);
468         sctx->is_dev_replace = is_dev_replace;
469         sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
470         sctx->curr = -1;
471         sctx->dev_root = dev->dev_root;
472         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
473                 struct scrub_bio *sbio;
474
475                 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
476                 if (!sbio)
477                         goto nomem;
478                 sctx->bios[i] = sbio;
479
480                 sbio->index = i;
481                 sbio->sctx = sctx;
482                 sbio->page_count = 0;
483                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
484                                 scrub_bio_end_io_worker, NULL, NULL);
485
486                 if (i != SCRUB_BIOS_PER_SCTX - 1)
487                         sctx->bios[i]->next_free = i + 1;
488                 else
489                         sctx->bios[i]->next_free = -1;
490         }
491         sctx->first_free = 0;
492         sctx->nodesize = dev->dev_root->nodesize;
493         sctx->sectorsize = dev->dev_root->sectorsize;
494         atomic_set(&sctx->bios_in_flight, 0);
495         atomic_set(&sctx->workers_pending, 0);
496         atomic_set(&sctx->cancel_req, 0);
497         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
498         INIT_LIST_HEAD(&sctx->csum_list);
499
500         spin_lock_init(&sctx->list_lock);
501         spin_lock_init(&sctx->stat_lock);
502         init_waitqueue_head(&sctx->list_wait);
503
504         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
505                                  fs_info->dev_replace.tgtdev, is_dev_replace);
506         if (ret) {
507                 scrub_free_ctx(sctx);
508                 return ERR_PTR(ret);
509         }
510         return sctx;
511
512 nomem:
513         scrub_free_ctx(sctx);
514         return ERR_PTR(-ENOMEM);
515 }
516
517 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
518                                      void *warn_ctx)
519 {
520         u64 isize;
521         u32 nlink;
522         int ret;
523         int i;
524         struct extent_buffer *eb;
525         struct btrfs_inode_item *inode_item;
526         struct scrub_warning *swarn = warn_ctx;
527         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
528         struct inode_fs_paths *ipath = NULL;
529         struct btrfs_root *local_root;
530         struct btrfs_key root_key;
531         struct btrfs_key key;
532
533         root_key.objectid = root;
534         root_key.type = BTRFS_ROOT_ITEM_KEY;
535         root_key.offset = (u64)-1;
536         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
537         if (IS_ERR(local_root)) {
538                 ret = PTR_ERR(local_root);
539                 goto err;
540         }
541
542         /*
543          * this makes the path point to (inum INODE_ITEM ioff)
544          */
545         key.objectid = inum;
546         key.type = BTRFS_INODE_ITEM_KEY;
547         key.offset = 0;
548
549         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
550         if (ret) {
551                 btrfs_release_path(swarn->path);
552                 goto err;
553         }
554
555         eb = swarn->path->nodes[0];
556         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
557                                         struct btrfs_inode_item);
558         isize = btrfs_inode_size(eb, inode_item);
559         nlink = btrfs_inode_nlink(eb, inode_item);
560         btrfs_release_path(swarn->path);
561
562         ipath = init_ipath(4096, local_root, swarn->path);
563         if (IS_ERR(ipath)) {
564                 ret = PTR_ERR(ipath);
565                 ipath = NULL;
566                 goto err;
567         }
568         ret = paths_from_inode(inum, ipath);
569
570         if (ret < 0)
571                 goto err;
572
573         /*
574          * we deliberately ignore the bit ipath might have been too small to
575          * hold all of the paths here
576          */
577         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
578                 btrfs_warn_in_rcu(fs_info, "%s at logical %llu on dev "
579                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
580                         "length %llu, links %u (path: %s)", swarn->errstr,
581                         swarn->logical, rcu_str_deref(swarn->dev->name),
582                         (unsigned long long)swarn->sector, root, inum, offset,
583                         min(isize - offset, (u64)PAGE_SIZE), nlink,
584                         (char *)(unsigned long)ipath->fspath->val[i]);
585
586         free_ipath(ipath);
587         return 0;
588
589 err:
590         btrfs_warn_in_rcu(fs_info, "%s at logical %llu on dev "
591                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
592                 "resolving failed with ret=%d", swarn->errstr,
593                 swarn->logical, rcu_str_deref(swarn->dev->name),
594                 (unsigned long long)swarn->sector, root, inum, offset, ret);
595
596         free_ipath(ipath);
597         return 0;
598 }
599
600 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
601 {
602         struct btrfs_device *dev;
603         struct btrfs_fs_info *fs_info;
604         struct btrfs_path *path;
605         struct btrfs_key found_key;
606         struct extent_buffer *eb;
607         struct btrfs_extent_item *ei;
608         struct scrub_warning swarn;
609         unsigned long ptr = 0;
610         u64 extent_item_pos;
611         u64 flags = 0;
612         u64 ref_root;
613         u32 item_size;
614         u8 ref_level = 0;
615         int ret;
616
617         WARN_ON(sblock->page_count < 1);
618         dev = sblock->pagev[0]->dev;
619         fs_info = sblock->sctx->dev_root->fs_info;
620
621         path = btrfs_alloc_path();
622         if (!path)
623                 return;
624
625         swarn.sector = (sblock->pagev[0]->physical) >> 9;
626         swarn.logical = sblock->pagev[0]->logical;
627         swarn.errstr = errstr;
628         swarn.dev = NULL;
629
630         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
631                                   &flags);
632         if (ret < 0)
633                 goto out;
634
635         extent_item_pos = swarn.logical - found_key.objectid;
636         swarn.extent_item_size = found_key.offset;
637
638         eb = path->nodes[0];
639         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
640         item_size = btrfs_item_size_nr(eb, path->slots[0]);
641
642         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
643                 do {
644                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
645                                                       item_size, &ref_root,
646                                                       &ref_level);
647                         btrfs_warn_in_rcu(fs_info,
648                                 "%s at logical %llu on dev %s, "
649                                 "sector %llu: metadata %s (level %d) in tree "
650                                 "%llu", errstr, swarn.logical,
651                                 rcu_str_deref(dev->name),
652                                 (unsigned long long)swarn.sector,
653                                 ref_level ? "node" : "leaf",
654                                 ret < 0 ? -1 : ref_level,
655                                 ret < 0 ? -1 : ref_root);
656                 } while (ret != 1);
657                 btrfs_release_path(path);
658         } else {
659                 btrfs_release_path(path);
660                 swarn.path = path;
661                 swarn.dev = dev;
662                 iterate_extent_inodes(fs_info, found_key.objectid,
663                                         extent_item_pos, 1,
664                                         scrub_print_warning_inode, &swarn);
665         }
666
667 out:
668         btrfs_free_path(path);
669 }
670
671 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
672 {
673         struct page *page = NULL;
674         unsigned long index;
675         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
676         int ret;
677         int corrected = 0;
678         struct btrfs_key key;
679         struct inode *inode = NULL;
680         struct btrfs_fs_info *fs_info;
681         u64 end = offset + PAGE_SIZE - 1;
682         struct btrfs_root *local_root;
683         int srcu_index;
684
685         key.objectid = root;
686         key.type = BTRFS_ROOT_ITEM_KEY;
687         key.offset = (u64)-1;
688
689         fs_info = fixup->root->fs_info;
690         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
691
692         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
693         if (IS_ERR(local_root)) {
694                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
695                 return PTR_ERR(local_root);
696         }
697
698         key.type = BTRFS_INODE_ITEM_KEY;
699         key.objectid = inum;
700         key.offset = 0;
701         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
702         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
703         if (IS_ERR(inode))
704                 return PTR_ERR(inode);
705
706         index = offset >> PAGE_SHIFT;
707
708         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
709         if (!page) {
710                 ret = -ENOMEM;
711                 goto out;
712         }
713
714         if (PageUptodate(page)) {
715                 if (PageDirty(page)) {
716                         /*
717                          * we need to write the data to the defect sector. the
718                          * data that was in that sector is not in memory,
719                          * because the page was modified. we must not write the
720                          * modified page to that sector.
721                          *
722                          * TODO: what could be done here: wait for the delalloc
723                          *       runner to write out that page (might involve
724                          *       COW) and see whether the sector is still
725                          *       referenced afterwards.
726                          *
727                          * For the meantime, we'll treat this error
728                          * incorrectable, although there is a chance that a
729                          * later scrub will find the bad sector again and that
730                          * there's no dirty page in memory, then.
731                          */
732                         ret = -EIO;
733                         goto out;
734                 }
735                 ret = repair_io_failure(inode, offset, PAGE_SIZE,
736                                         fixup->logical, page,
737                                         offset - page_offset(page),
738                                         fixup->mirror_num);
739                 unlock_page(page);
740                 corrected = !ret;
741         } else {
742                 /*
743                  * we need to get good data first. the general readpage path
744                  * will call repair_io_failure for us, we just have to make
745                  * sure we read the bad mirror.
746                  */
747                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
748                                         EXTENT_DAMAGED);
749                 if (ret) {
750                         /* set_extent_bits should give proper error */
751                         WARN_ON(ret > 0);
752                         if (ret > 0)
753                                 ret = -EFAULT;
754                         goto out;
755                 }
756
757                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
758                                                 btrfs_get_extent,
759                                                 fixup->mirror_num);
760                 wait_on_page_locked(page);
761
762                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
763                                                 end, EXTENT_DAMAGED, 0, NULL);
764                 if (!corrected)
765                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
766                                                 EXTENT_DAMAGED);
767         }
768
769 out:
770         if (page)
771                 put_page(page);
772
773         iput(inode);
774
775         if (ret < 0)
776                 return ret;
777
778         if (ret == 0 && corrected) {
779                 /*
780                  * we only need to call readpage for one of the inodes belonging
781                  * to this extent. so make iterate_extent_inodes stop
782                  */
783                 return 1;
784         }
785
786         return -EIO;
787 }
788
789 static void scrub_fixup_nodatasum(struct btrfs_work *work)
790 {
791         int ret;
792         struct scrub_fixup_nodatasum *fixup;
793         struct scrub_ctx *sctx;
794         struct btrfs_trans_handle *trans = NULL;
795         struct btrfs_path *path;
796         int uncorrectable = 0;
797
798         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
799         sctx = fixup->sctx;
800
801         path = btrfs_alloc_path();
802         if (!path) {
803                 spin_lock(&sctx->stat_lock);
804                 ++sctx->stat.malloc_errors;
805                 spin_unlock(&sctx->stat_lock);
806                 uncorrectable = 1;
807                 goto out;
808         }
809
810         trans = btrfs_join_transaction(fixup->root);
811         if (IS_ERR(trans)) {
812                 uncorrectable = 1;
813                 goto out;
814         }
815
816         /*
817          * the idea is to trigger a regular read through the standard path. we
818          * read a page from the (failed) logical address by specifying the
819          * corresponding copynum of the failed sector. thus, that readpage is
820          * expected to fail.
821          * that is the point where on-the-fly error correction will kick in
822          * (once it's finished) and rewrite the failed sector if a good copy
823          * can be found.
824          */
825         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
826                                                 path, scrub_fixup_readpage,
827                                                 fixup);
828         if (ret < 0) {
829                 uncorrectable = 1;
830                 goto out;
831         }
832         WARN_ON(ret != 1);
833
834         spin_lock(&sctx->stat_lock);
835         ++sctx->stat.corrected_errors;
836         spin_unlock(&sctx->stat_lock);
837
838 out:
839         if (trans && !IS_ERR(trans))
840                 btrfs_end_transaction(trans, fixup->root);
841         if (uncorrectable) {
842                 spin_lock(&sctx->stat_lock);
843                 ++sctx->stat.uncorrectable_errors;
844                 spin_unlock(&sctx->stat_lock);
845                 btrfs_dev_replace_stats_inc(
846                         &sctx->dev_root->fs_info->dev_replace.
847                         num_uncorrectable_read_errors);
848                 btrfs_err_rl_in_rcu(sctx->dev_root->fs_info,
849                     "unable to fixup (nodatasum) error at logical %llu on dev %s",
850                         fixup->logical, rcu_str_deref(fixup->dev->name));
851         }
852
853         btrfs_free_path(path);
854         kfree(fixup);
855
856         scrub_pending_trans_workers_dec(sctx);
857 }
858
859 static inline void scrub_get_recover(struct scrub_recover *recover)
860 {
861         atomic_inc(&recover->refs);
862 }
863
864 static inline void scrub_put_recover(struct scrub_recover *recover)
865 {
866         if (atomic_dec_and_test(&recover->refs)) {
867                 btrfs_put_bbio(recover->bbio);
868                 kfree(recover);
869         }
870 }
871
872 /*
873  * scrub_handle_errored_block gets called when either verification of the
874  * pages failed or the bio failed to read, e.g. with EIO. In the latter
875  * case, this function handles all pages in the bio, even though only one
876  * may be bad.
877  * The goal of this function is to repair the errored block by using the
878  * contents of one of the mirrors.
879  */
880 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
881 {
882         struct scrub_ctx *sctx = sblock_to_check->sctx;
883         struct btrfs_device *dev;
884         struct btrfs_fs_info *fs_info;
885         u64 length;
886         u64 logical;
887         unsigned int failed_mirror_index;
888         unsigned int is_metadata;
889         unsigned int have_csum;
890         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
891         struct scrub_block *sblock_bad;
892         int ret;
893         int mirror_index;
894         int page_num;
895         int success;
896         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
897                                       DEFAULT_RATELIMIT_BURST);
898
899         BUG_ON(sblock_to_check->page_count < 1);
900         fs_info = sctx->dev_root->fs_info;
901         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
902                 /*
903                  * if we find an error in a super block, we just report it.
904                  * They will get written with the next transaction commit
905                  * anyway
906                  */
907                 spin_lock(&sctx->stat_lock);
908                 ++sctx->stat.super_errors;
909                 spin_unlock(&sctx->stat_lock);
910                 return 0;
911         }
912         length = sblock_to_check->page_count * PAGE_SIZE;
913         logical = sblock_to_check->pagev[0]->logical;
914         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
915         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
916         is_metadata = !(sblock_to_check->pagev[0]->flags &
917                         BTRFS_EXTENT_FLAG_DATA);
918         have_csum = sblock_to_check->pagev[0]->have_csum;
919         dev = sblock_to_check->pagev[0]->dev;
920
921         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
922                 sblocks_for_recheck = NULL;
923                 goto nodatasum_case;
924         }
925
926         /*
927          * read all mirrors one after the other. This includes to
928          * re-read the extent or metadata block that failed (that was
929          * the cause that this fixup code is called) another time,
930          * page by page this time in order to know which pages
931          * caused I/O errors and which ones are good (for all mirrors).
932          * It is the goal to handle the situation when more than one
933          * mirror contains I/O errors, but the errors do not
934          * overlap, i.e. the data can be repaired by selecting the
935          * pages from those mirrors without I/O error on the
936          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
937          * would be that mirror #1 has an I/O error on the first page,
938          * the second page is good, and mirror #2 has an I/O error on
939          * the second page, but the first page is good.
940          * Then the first page of the first mirror can be repaired by
941          * taking the first page of the second mirror, and the
942          * second page of the second mirror can be repaired by
943          * copying the contents of the 2nd page of the 1st mirror.
944          * One more note: if the pages of one mirror contain I/O
945          * errors, the checksum cannot be verified. In order to get
946          * the best data for repairing, the first attempt is to find
947          * a mirror without I/O errors and with a validated checksum.
948          * Only if this is not possible, the pages are picked from
949          * mirrors with I/O errors without considering the checksum.
950          * If the latter is the case, at the end, the checksum of the
951          * repaired area is verified in order to correctly maintain
952          * the statistics.
953          */
954
955         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
956                                       sizeof(*sblocks_for_recheck), GFP_NOFS);
957         if (!sblocks_for_recheck) {
958                 spin_lock(&sctx->stat_lock);
959                 sctx->stat.malloc_errors++;
960                 sctx->stat.read_errors++;
961                 sctx->stat.uncorrectable_errors++;
962                 spin_unlock(&sctx->stat_lock);
963                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
964                 goto out;
965         }
966
967         /* setup the context, map the logical blocks and alloc the pages */
968         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
969         if (ret) {
970                 spin_lock(&sctx->stat_lock);
971                 sctx->stat.read_errors++;
972                 sctx->stat.uncorrectable_errors++;
973                 spin_unlock(&sctx->stat_lock);
974                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
975                 goto out;
976         }
977         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
978         sblock_bad = sblocks_for_recheck + failed_mirror_index;
979
980         /* build and submit the bios for the failed mirror, check checksums */
981         scrub_recheck_block(fs_info, sblock_bad, 1);
982
983         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
984             sblock_bad->no_io_error_seen) {
985                 /*
986                  * the error disappeared after reading page by page, or
987                  * the area was part of a huge bio and other parts of the
988                  * bio caused I/O errors, or the block layer merged several
989                  * read requests into one and the error is caused by a
990                  * different bio (usually one of the two latter cases is
991                  * the cause)
992                  */
993                 spin_lock(&sctx->stat_lock);
994                 sctx->stat.unverified_errors++;
995                 sblock_to_check->data_corrected = 1;
996                 spin_unlock(&sctx->stat_lock);
997
998                 if (sctx->is_dev_replace)
999                         scrub_write_block_to_dev_replace(sblock_bad);
1000                 goto out;
1001         }
1002
1003         if (!sblock_bad->no_io_error_seen) {
1004                 spin_lock(&sctx->stat_lock);
1005                 sctx->stat.read_errors++;
1006                 spin_unlock(&sctx->stat_lock);
1007                 if (__ratelimit(&_rs))
1008                         scrub_print_warning("i/o error", sblock_to_check);
1009                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1010         } else if (sblock_bad->checksum_error) {
1011                 spin_lock(&sctx->stat_lock);
1012                 sctx->stat.csum_errors++;
1013                 spin_unlock(&sctx->stat_lock);
1014                 if (__ratelimit(&_rs))
1015                         scrub_print_warning("checksum error", sblock_to_check);
1016                 btrfs_dev_stat_inc_and_print(dev,
1017                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
1018         } else if (sblock_bad->header_error) {
1019                 spin_lock(&sctx->stat_lock);
1020                 sctx->stat.verify_errors++;
1021                 spin_unlock(&sctx->stat_lock);
1022                 if (__ratelimit(&_rs))
1023                         scrub_print_warning("checksum/header error",
1024                                             sblock_to_check);
1025                 if (sblock_bad->generation_error)
1026                         btrfs_dev_stat_inc_and_print(dev,
1027                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1028                 else
1029                         btrfs_dev_stat_inc_and_print(dev,
1030                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1031         }
1032
1033         if (sctx->readonly) {
1034                 ASSERT(!sctx->is_dev_replace);
1035                 goto out;
1036         }
1037
1038         if (!is_metadata && !have_csum) {
1039                 struct scrub_fixup_nodatasum *fixup_nodatasum;
1040
1041                 WARN_ON(sctx->is_dev_replace);
1042
1043 nodatasum_case:
1044
1045                 /*
1046                  * !is_metadata and !have_csum, this means that the data
1047                  * might not be COWed, that it might be modified
1048                  * concurrently. The general strategy to work on the
1049                  * commit root does not help in the case when COW is not
1050                  * used.
1051                  */
1052                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1053                 if (!fixup_nodatasum)
1054                         goto did_not_correct_error;
1055                 fixup_nodatasum->sctx = sctx;
1056                 fixup_nodatasum->dev = dev;
1057                 fixup_nodatasum->logical = logical;
1058                 fixup_nodatasum->root = fs_info->extent_root;
1059                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1060                 scrub_pending_trans_workers_inc(sctx);
1061                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1062                                 scrub_fixup_nodatasum, NULL, NULL);
1063                 btrfs_queue_work(fs_info->scrub_workers,
1064                                  &fixup_nodatasum->work);
1065                 goto out;
1066         }
1067
1068         /*
1069          * now build and submit the bios for the other mirrors, check
1070          * checksums.
1071          * First try to pick the mirror which is completely without I/O
1072          * errors and also does not have a checksum error.
1073          * If one is found, and if a checksum is present, the full block
1074          * that is known to contain an error is rewritten. Afterwards
1075          * the block is known to be corrected.
1076          * If a mirror is found which is completely correct, and no
1077          * checksum is present, only those pages are rewritten that had
1078          * an I/O error in the block to be repaired, since it cannot be
1079          * determined, which copy of the other pages is better (and it
1080          * could happen otherwise that a correct page would be
1081          * overwritten by a bad one).
1082          */
1083         for (mirror_index = 0;
1084              mirror_index < BTRFS_MAX_MIRRORS &&
1085              sblocks_for_recheck[mirror_index].page_count > 0;
1086              mirror_index++) {
1087                 struct scrub_block *sblock_other;
1088
1089                 if (mirror_index == failed_mirror_index)
1090                         continue;
1091                 sblock_other = sblocks_for_recheck + mirror_index;
1092
1093                 /* build and submit the bios, check checksums */
1094                 scrub_recheck_block(fs_info, sblock_other, 0);
1095
1096                 if (!sblock_other->header_error &&
1097                     !sblock_other->checksum_error &&
1098                     sblock_other->no_io_error_seen) {
1099                         if (sctx->is_dev_replace) {
1100                                 scrub_write_block_to_dev_replace(sblock_other);
1101                                 goto corrected_error;
1102                         } else {
1103                                 ret = scrub_repair_block_from_good_copy(
1104                                                 sblock_bad, sblock_other);
1105                                 if (!ret)
1106                                         goto corrected_error;
1107                         }
1108                 }
1109         }
1110
1111         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1112                 goto did_not_correct_error;
1113
1114         /*
1115          * In case of I/O errors in the area that is supposed to be
1116          * repaired, continue by picking good copies of those pages.
1117          * Select the good pages from mirrors to rewrite bad pages from
1118          * the area to fix. Afterwards verify the checksum of the block
1119          * that is supposed to be repaired. This verification step is
1120          * only done for the purpose of statistic counting and for the
1121          * final scrub report, whether errors remain.
1122          * A perfect algorithm could make use of the checksum and try
1123          * all possible combinations of pages from the different mirrors
1124          * until the checksum verification succeeds. For example, when
1125          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1126          * of mirror #2 is readable but the final checksum test fails,
1127          * then the 2nd page of mirror #3 could be tried, whether now
1128          * the final checksum succeeds. But this would be a rare
1129          * exception and is therefore not implemented. At least it is
1130          * avoided that the good copy is overwritten.
1131          * A more useful improvement would be to pick the sectors
1132          * without I/O error based on sector sizes (512 bytes on legacy
1133          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1134          * mirror could be repaired by taking 512 byte of a different
1135          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1136          * area are unreadable.
1137          */
1138         success = 1;
1139         for (page_num = 0; page_num < sblock_bad->page_count;
1140              page_num++) {
1141                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1142                 struct scrub_block *sblock_other = NULL;
1143
1144                 /* skip no-io-error page in scrub */
1145                 if (!page_bad->io_error && !sctx->is_dev_replace)
1146                         continue;
1147
1148                 /* try to find no-io-error page in mirrors */
1149                 if (page_bad->io_error) {
1150                         for (mirror_index = 0;
1151                              mirror_index < BTRFS_MAX_MIRRORS &&
1152                              sblocks_for_recheck[mirror_index].page_count > 0;
1153                              mirror_index++) {
1154                                 if (!sblocks_for_recheck[mirror_index].
1155                                     pagev[page_num]->io_error) {
1156                                         sblock_other = sblocks_for_recheck +
1157                                                        mirror_index;
1158                                         break;
1159                                 }
1160                         }
1161                         if (!sblock_other)
1162                                 success = 0;
1163                 }
1164
1165                 if (sctx->is_dev_replace) {
1166                         /*
1167                          * did not find a mirror to fetch the page
1168                          * from. scrub_write_page_to_dev_replace()
1169                          * handles this case (page->io_error), by
1170                          * filling the block with zeros before
1171                          * submitting the write request
1172                          */
1173                         if (!sblock_other)
1174                                 sblock_other = sblock_bad;
1175
1176                         if (scrub_write_page_to_dev_replace(sblock_other,
1177                                                             page_num) != 0) {
1178                                 btrfs_dev_replace_stats_inc(
1179                                         &sctx->dev_root->
1180                                         fs_info->dev_replace.
1181                                         num_write_errors);
1182                                 success = 0;
1183                         }
1184                 } else if (sblock_other) {
1185                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1186                                                                sblock_other,
1187                                                                page_num, 0);
1188                         if (0 == ret)
1189                                 page_bad->io_error = 0;
1190                         else
1191                                 success = 0;
1192                 }
1193         }
1194
1195         if (success && !sctx->is_dev_replace) {
1196                 if (is_metadata || have_csum) {
1197                         /*
1198                          * need to verify the checksum now that all
1199                          * sectors on disk are repaired (the write
1200                          * request for data to be repaired is on its way).
1201                          * Just be lazy and use scrub_recheck_block()
1202                          * which re-reads the data before the checksum
1203                          * is verified, but most likely the data comes out
1204                          * of the page cache.
1205                          */
1206                         scrub_recheck_block(fs_info, sblock_bad, 1);
1207                         if (!sblock_bad->header_error &&
1208                             !sblock_bad->checksum_error &&
1209                             sblock_bad->no_io_error_seen)
1210                                 goto corrected_error;
1211                         else
1212                                 goto did_not_correct_error;
1213                 } else {
1214 corrected_error:
1215                         spin_lock(&sctx->stat_lock);
1216                         sctx->stat.corrected_errors++;
1217                         sblock_to_check->data_corrected = 1;
1218                         spin_unlock(&sctx->stat_lock);
1219                         btrfs_err_rl_in_rcu(fs_info,
1220                                 "fixed up error at logical %llu on dev %s",
1221                                 logical, rcu_str_deref(dev->name));
1222                 }
1223         } else {
1224 did_not_correct_error:
1225                 spin_lock(&sctx->stat_lock);
1226                 sctx->stat.uncorrectable_errors++;
1227                 spin_unlock(&sctx->stat_lock);
1228                 btrfs_err_rl_in_rcu(fs_info,
1229                         "unable to fixup (regular) error at logical %llu on dev %s",
1230                         logical, rcu_str_deref(dev->name));
1231         }
1232
1233 out:
1234         if (sblocks_for_recheck) {
1235                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1236                      mirror_index++) {
1237                         struct scrub_block *sblock = sblocks_for_recheck +
1238                                                      mirror_index;
1239                         struct scrub_recover *recover;
1240                         int page_index;
1241
1242                         for (page_index = 0; page_index < sblock->page_count;
1243                              page_index++) {
1244                                 sblock->pagev[page_index]->sblock = NULL;
1245                                 recover = sblock->pagev[page_index]->recover;
1246                                 if (recover) {
1247                                         scrub_put_recover(recover);
1248                                         sblock->pagev[page_index]->recover =
1249                                                                         NULL;
1250                                 }
1251                                 scrub_page_put(sblock->pagev[page_index]);
1252                         }
1253                 }
1254                 kfree(sblocks_for_recheck);
1255         }
1256
1257         return 0;
1258 }
1259
1260 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1261 {
1262         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1263                 return 2;
1264         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1265                 return 3;
1266         else
1267                 return (int)bbio->num_stripes;
1268 }
1269
1270 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1271                                                  u64 *raid_map,
1272                                                  u64 mapped_length,
1273                                                  int nstripes, int mirror,
1274                                                  int *stripe_index,
1275                                                  u64 *stripe_offset)
1276 {
1277         int i;
1278
1279         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1280                 /* RAID5/6 */
1281                 for (i = 0; i < nstripes; i++) {
1282                         if (raid_map[i] == RAID6_Q_STRIPE ||
1283                             raid_map[i] == RAID5_P_STRIPE)
1284                                 continue;
1285
1286                         if (logical >= raid_map[i] &&
1287                             logical < raid_map[i] + mapped_length)
1288                                 break;
1289                 }
1290
1291                 *stripe_index = i;
1292                 *stripe_offset = logical - raid_map[i];
1293         } else {
1294                 /* The other RAID type */
1295                 *stripe_index = mirror;
1296                 *stripe_offset = 0;
1297         }
1298 }
1299
1300 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1301                                      struct scrub_block *sblocks_for_recheck)
1302 {
1303         struct scrub_ctx *sctx = original_sblock->sctx;
1304         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
1305         u64 length = original_sblock->page_count * PAGE_SIZE;
1306         u64 logical = original_sblock->pagev[0]->logical;
1307         u64 generation = original_sblock->pagev[0]->generation;
1308         u64 flags = original_sblock->pagev[0]->flags;
1309         u64 have_csum = original_sblock->pagev[0]->have_csum;
1310         struct scrub_recover *recover;
1311         struct btrfs_bio *bbio;
1312         u64 sublen;
1313         u64 mapped_length;
1314         u64 stripe_offset;
1315         int stripe_index;
1316         int page_index = 0;
1317         int mirror_index;
1318         int nmirrors;
1319         int ret;
1320
1321         /*
1322          * note: the two members refs and outstanding_pages
1323          * are not used (and not set) in the blocks that are used for
1324          * the recheck procedure
1325          */
1326
1327         while (length > 0) {
1328                 sublen = min_t(u64, length, PAGE_SIZE);
1329                 mapped_length = sublen;
1330                 bbio = NULL;
1331
1332                 /*
1333                  * with a length of PAGE_SIZE, each returned stripe
1334                  * represents one mirror
1335                  */
1336                 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
1337                                        &mapped_length, &bbio, 0, 1);
1338                 if (ret || !bbio || mapped_length < sublen) {
1339                         btrfs_put_bbio(bbio);
1340                         return -EIO;
1341                 }
1342
1343                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1344                 if (!recover) {
1345                         btrfs_put_bbio(bbio);
1346                         return -ENOMEM;
1347                 }
1348
1349                 atomic_set(&recover->refs, 1);
1350                 recover->bbio = bbio;
1351                 recover->map_length = mapped_length;
1352
1353                 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1354
1355                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1356
1357                 for (mirror_index = 0; mirror_index < nmirrors;
1358                      mirror_index++) {
1359                         struct scrub_block *sblock;
1360                         struct scrub_page *page;
1361
1362                         sblock = sblocks_for_recheck + mirror_index;
1363                         sblock->sctx = sctx;
1364
1365                         page = kzalloc(sizeof(*page), GFP_NOFS);
1366                         if (!page) {
1367 leave_nomem:
1368                                 spin_lock(&sctx->stat_lock);
1369                                 sctx->stat.malloc_errors++;
1370                                 spin_unlock(&sctx->stat_lock);
1371                                 scrub_put_recover(recover);
1372                                 return -ENOMEM;
1373                         }
1374                         scrub_page_get(page);
1375                         sblock->pagev[page_index] = page;
1376                         page->sblock = sblock;
1377                         page->flags = flags;
1378                         page->generation = generation;
1379                         page->logical = logical;
1380                         page->have_csum = have_csum;
1381                         if (have_csum)
1382                                 memcpy(page->csum,
1383                                        original_sblock->pagev[0]->csum,
1384                                        sctx->csum_size);
1385
1386                         scrub_stripe_index_and_offset(logical,
1387                                                       bbio->map_type,
1388                                                       bbio->raid_map,
1389                                                       mapped_length,
1390                                                       bbio->num_stripes -
1391                                                       bbio->num_tgtdevs,
1392                                                       mirror_index,
1393                                                       &stripe_index,
1394                                                       &stripe_offset);
1395                         page->physical = bbio->stripes[stripe_index].physical +
1396                                          stripe_offset;
1397                         page->dev = bbio->stripes[stripe_index].dev;
1398
1399                         BUG_ON(page_index >= original_sblock->page_count);
1400                         page->physical_for_dev_replace =
1401                                 original_sblock->pagev[page_index]->
1402                                 physical_for_dev_replace;
1403                         /* for missing devices, dev->bdev is NULL */
1404                         page->mirror_num = mirror_index + 1;
1405                         sblock->page_count++;
1406                         page->page = alloc_page(GFP_NOFS);
1407                         if (!page->page)
1408                                 goto leave_nomem;
1409
1410                         scrub_get_recover(recover);
1411                         page->recover = recover;
1412                 }
1413                 scrub_put_recover(recover);
1414                 length -= sublen;
1415                 logical += sublen;
1416                 page_index++;
1417         }
1418
1419         return 0;
1420 }
1421
1422 struct scrub_bio_ret {
1423         struct completion event;
1424         int error;
1425 };
1426
1427 static void scrub_bio_wait_endio(struct bio *bio)
1428 {
1429         struct scrub_bio_ret *ret = bio->bi_private;
1430
1431         ret->error = bio->bi_error;
1432         complete(&ret->event);
1433 }
1434
1435 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1436 {
1437         return page->recover &&
1438                (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
1439 }
1440
1441 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1442                                         struct bio *bio,
1443                                         struct scrub_page *page)
1444 {
1445         struct scrub_bio_ret done;
1446         int ret;
1447
1448         init_completion(&done.event);
1449         done.error = 0;
1450         bio->bi_iter.bi_sector = page->logical >> 9;
1451         bio->bi_private = &done;
1452         bio->bi_end_io = scrub_bio_wait_endio;
1453
1454         ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
1455                                     page->recover->map_length,
1456                                     page->mirror_num, 0);
1457         if (ret)
1458                 return ret;
1459
1460         wait_for_completion(&done.event);
1461         if (done.error)
1462                 return -EIO;
1463
1464         return 0;
1465 }
1466
1467 /*
1468  * this function will check the on disk data for checksum errors, header
1469  * errors and read I/O errors. If any I/O errors happen, the exact pages
1470  * which are errored are marked as being bad. The goal is to enable scrub
1471  * to take those pages that are not errored from all the mirrors so that
1472  * the pages that are errored in the just handled mirror can be repaired.
1473  */
1474 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1475                                 struct scrub_block *sblock,
1476                                 int retry_failed_mirror)
1477 {
1478         int page_num;
1479
1480         sblock->no_io_error_seen = 1;
1481
1482         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1483                 struct bio *bio;
1484                 struct scrub_page *page = sblock->pagev[page_num];
1485
1486                 if (page->dev->bdev == NULL) {
1487                         page->io_error = 1;
1488                         sblock->no_io_error_seen = 0;
1489                         continue;
1490                 }
1491
1492                 WARN_ON(!page->page);
1493                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1494                 if (!bio) {
1495                         page->io_error = 1;
1496                         sblock->no_io_error_seen = 0;
1497                         continue;
1498                 }
1499                 bio->bi_bdev = page->dev->bdev;
1500
1501                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1502                 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1503                         if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1504                                 sblock->no_io_error_seen = 0;
1505                 } else {
1506                         bio->bi_iter.bi_sector = page->physical >> 9;
1507                         bio_set_op_attrs(bio, REQ_OP_READ, 0);
1508
1509                         if (btrfsic_submit_bio_wait(bio))
1510                                 sblock->no_io_error_seen = 0;
1511                 }
1512
1513                 bio_put(bio);
1514         }
1515
1516         if (sblock->no_io_error_seen)
1517                 scrub_recheck_block_checksum(sblock);
1518 }
1519
1520 static inline int scrub_check_fsid(u8 fsid[],
1521                                    struct scrub_page *spage)
1522 {
1523         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1524         int ret;
1525
1526         ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1527         return !ret;
1528 }
1529
1530 static void scrub_recheck_block_checksum(struct scrub_block *sblock)
1531 {
1532         sblock->header_error = 0;
1533         sblock->checksum_error = 0;
1534         sblock->generation_error = 0;
1535
1536         if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
1537                 scrub_checksum_data(sblock);
1538         else
1539                 scrub_checksum_tree_block(sblock);
1540 }
1541
1542 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1543                                              struct scrub_block *sblock_good)
1544 {
1545         int page_num;
1546         int ret = 0;
1547
1548         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1549                 int ret_sub;
1550
1551                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1552                                                            sblock_good,
1553                                                            page_num, 1);
1554                 if (ret_sub)
1555                         ret = ret_sub;
1556         }
1557
1558         return ret;
1559 }
1560
1561 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1562                                             struct scrub_block *sblock_good,
1563                                             int page_num, int force_write)
1564 {
1565         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1566         struct scrub_page *page_good = sblock_good->pagev[page_num];
1567
1568         BUG_ON(page_bad->page == NULL);
1569         BUG_ON(page_good->page == NULL);
1570         if (force_write || sblock_bad->header_error ||
1571             sblock_bad->checksum_error || page_bad->io_error) {
1572                 struct bio *bio;
1573                 int ret;
1574
1575                 if (!page_bad->dev->bdev) {
1576                         btrfs_warn_rl(sblock_bad->sctx->dev_root->fs_info,
1577                                 "scrub_repair_page_from_good_copy(bdev == NULL) "
1578                                 "is unexpected");
1579                         return -EIO;
1580                 }
1581
1582                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1583                 if (!bio)
1584                         return -EIO;
1585                 bio->bi_bdev = page_bad->dev->bdev;
1586                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1587                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1588
1589                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1590                 if (PAGE_SIZE != ret) {
1591                         bio_put(bio);
1592                         return -EIO;
1593                 }
1594
1595                 if (btrfsic_submit_bio_wait(bio)) {
1596                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1597                                 BTRFS_DEV_STAT_WRITE_ERRS);
1598                         btrfs_dev_replace_stats_inc(
1599                                 &sblock_bad->sctx->dev_root->fs_info->
1600                                 dev_replace.num_write_errors);
1601                         bio_put(bio);
1602                         return -EIO;
1603                 }
1604                 bio_put(bio);
1605         }
1606
1607         return 0;
1608 }
1609
1610 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1611 {
1612         int page_num;
1613
1614         /*
1615          * This block is used for the check of the parity on the source device,
1616          * so the data needn't be written into the destination device.
1617          */
1618         if (sblock->sparity)
1619                 return;
1620
1621         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1622                 int ret;
1623
1624                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1625                 if (ret)
1626                         btrfs_dev_replace_stats_inc(
1627                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1628                                 num_write_errors);
1629         }
1630 }
1631
1632 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1633                                            int page_num)
1634 {
1635         struct scrub_page *spage = sblock->pagev[page_num];
1636
1637         BUG_ON(spage->page == NULL);
1638         if (spage->io_error) {
1639                 void *mapped_buffer = kmap_atomic(spage->page);
1640
1641                 memset(mapped_buffer, 0, PAGE_SIZE);
1642                 flush_dcache_page(spage->page);
1643                 kunmap_atomic(mapped_buffer);
1644         }
1645         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1646 }
1647
1648 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1649                                     struct scrub_page *spage)
1650 {
1651         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1652         struct scrub_bio *sbio;
1653         int ret;
1654
1655         mutex_lock(&wr_ctx->wr_lock);
1656 again:
1657         if (!wr_ctx->wr_curr_bio) {
1658                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1659                                               GFP_KERNEL);
1660                 if (!wr_ctx->wr_curr_bio) {
1661                         mutex_unlock(&wr_ctx->wr_lock);
1662                         return -ENOMEM;
1663                 }
1664                 wr_ctx->wr_curr_bio->sctx = sctx;
1665                 wr_ctx->wr_curr_bio->page_count = 0;
1666         }
1667         sbio = wr_ctx->wr_curr_bio;
1668         if (sbio->page_count == 0) {
1669                 struct bio *bio;
1670
1671                 sbio->physical = spage->physical_for_dev_replace;
1672                 sbio->logical = spage->logical;
1673                 sbio->dev = wr_ctx->tgtdev;
1674                 bio = sbio->bio;
1675                 if (!bio) {
1676                         bio = btrfs_io_bio_alloc(GFP_KERNEL,
1677                                         wr_ctx->pages_per_wr_bio);
1678                         if (!bio) {
1679                                 mutex_unlock(&wr_ctx->wr_lock);
1680                                 return -ENOMEM;
1681                         }
1682                         sbio->bio = bio;
1683                 }
1684
1685                 bio->bi_private = sbio;
1686                 bio->bi_end_io = scrub_wr_bio_end_io;
1687                 bio->bi_bdev = sbio->dev->bdev;
1688                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1689                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1690                 sbio->err = 0;
1691         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1692                    spage->physical_for_dev_replace ||
1693                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1694                    spage->logical) {
1695                 scrub_wr_submit(sctx);
1696                 goto again;
1697         }
1698
1699         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1700         if (ret != PAGE_SIZE) {
1701                 if (sbio->page_count < 1) {
1702                         bio_put(sbio->bio);
1703                         sbio->bio = NULL;
1704                         mutex_unlock(&wr_ctx->wr_lock);
1705                         return -EIO;
1706                 }
1707                 scrub_wr_submit(sctx);
1708                 goto again;
1709         }
1710
1711         sbio->pagev[sbio->page_count] = spage;
1712         scrub_page_get(spage);
1713         sbio->page_count++;
1714         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1715                 scrub_wr_submit(sctx);
1716         mutex_unlock(&wr_ctx->wr_lock);
1717
1718         return 0;
1719 }
1720
1721 static void scrub_wr_submit(struct scrub_ctx *sctx)
1722 {
1723         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1724         struct scrub_bio *sbio;
1725
1726         if (!wr_ctx->wr_curr_bio)
1727                 return;
1728
1729         sbio = wr_ctx->wr_curr_bio;
1730         wr_ctx->wr_curr_bio = NULL;
1731         WARN_ON(!sbio->bio->bi_bdev);
1732         scrub_pending_bio_inc(sctx);
1733         /* process all writes in a single worker thread. Then the block layer
1734          * orders the requests before sending them to the driver which
1735          * doubled the write performance on spinning disks when measured
1736          * with Linux 3.5 */
1737         btrfsic_submit_bio(sbio->bio);
1738 }
1739
1740 static void scrub_wr_bio_end_io(struct bio *bio)
1741 {
1742         struct scrub_bio *sbio = bio->bi_private;
1743         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1744
1745         sbio->err = bio->bi_error;
1746         sbio->bio = bio;
1747
1748         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1749                          scrub_wr_bio_end_io_worker, NULL, NULL);
1750         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1751 }
1752
1753 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1754 {
1755         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1756         struct scrub_ctx *sctx = sbio->sctx;
1757         int i;
1758
1759         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1760         if (sbio->err) {
1761                 struct btrfs_dev_replace *dev_replace =
1762                         &sbio->sctx->dev_root->fs_info->dev_replace;
1763
1764                 for (i = 0; i < sbio->page_count; i++) {
1765                         struct scrub_page *spage = sbio->pagev[i];
1766
1767                         spage->io_error = 1;
1768                         btrfs_dev_replace_stats_inc(&dev_replace->
1769                                                     num_write_errors);
1770                 }
1771         }
1772
1773         for (i = 0; i < sbio->page_count; i++)
1774                 scrub_page_put(sbio->pagev[i]);
1775
1776         bio_put(sbio->bio);
1777         kfree(sbio);
1778         scrub_pending_bio_dec(sctx);
1779 }
1780
1781 static int scrub_checksum(struct scrub_block *sblock)
1782 {
1783         u64 flags;
1784         int ret;
1785
1786         /*
1787          * No need to initialize these stats currently,
1788          * because this function only use return value
1789          * instead of these stats value.
1790          *
1791          * Todo:
1792          * always use stats
1793          */
1794         sblock->header_error = 0;
1795         sblock->generation_error = 0;
1796         sblock->checksum_error = 0;
1797
1798         WARN_ON(sblock->page_count < 1);
1799         flags = sblock->pagev[0]->flags;
1800         ret = 0;
1801         if (flags & BTRFS_EXTENT_FLAG_DATA)
1802                 ret = scrub_checksum_data(sblock);
1803         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1804                 ret = scrub_checksum_tree_block(sblock);
1805         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1806                 (void)scrub_checksum_super(sblock);
1807         else
1808                 WARN_ON(1);
1809         if (ret)
1810                 scrub_handle_errored_block(sblock);
1811
1812         return ret;
1813 }
1814
1815 static int scrub_checksum_data(struct scrub_block *sblock)
1816 {
1817         struct scrub_ctx *sctx = sblock->sctx;
1818         u8 csum[BTRFS_CSUM_SIZE];
1819         u8 *on_disk_csum;
1820         struct page *page;
1821         void *buffer;
1822         u32 crc = ~(u32)0;
1823         u64 len;
1824         int index;
1825
1826         BUG_ON(sblock->page_count < 1);
1827         if (!sblock->pagev[0]->have_csum)
1828                 return 0;
1829
1830         on_disk_csum = sblock->pagev[0]->csum;
1831         page = sblock->pagev[0]->page;
1832         buffer = kmap_atomic(page);
1833
1834         len = sctx->sectorsize;
1835         index = 0;
1836         for (;;) {
1837                 u64 l = min_t(u64, len, PAGE_SIZE);
1838
1839                 crc = btrfs_csum_data(buffer, crc, l);
1840                 kunmap_atomic(buffer);
1841                 len -= l;
1842                 if (len == 0)
1843                         break;
1844                 index++;
1845                 BUG_ON(index >= sblock->page_count);
1846                 BUG_ON(!sblock->pagev[index]->page);
1847                 page = sblock->pagev[index]->page;
1848                 buffer = kmap_atomic(page);
1849         }
1850
1851         btrfs_csum_final(crc, csum);
1852         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1853                 sblock->checksum_error = 1;
1854
1855         return sblock->checksum_error;
1856 }
1857
1858 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1859 {
1860         struct scrub_ctx *sctx = sblock->sctx;
1861         struct btrfs_header *h;
1862         struct btrfs_root *root = sctx->dev_root;
1863         struct btrfs_fs_info *fs_info = root->fs_info;
1864         u8 calculated_csum[BTRFS_CSUM_SIZE];
1865         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1866         struct page *page;
1867         void *mapped_buffer;
1868         u64 mapped_size;
1869         void *p;
1870         u32 crc = ~(u32)0;
1871         u64 len;
1872         int index;
1873
1874         BUG_ON(sblock->page_count < 1);
1875         page = sblock->pagev[0]->page;
1876         mapped_buffer = kmap_atomic(page);
1877         h = (struct btrfs_header *)mapped_buffer;
1878         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1879
1880         /*
1881          * we don't use the getter functions here, as we
1882          * a) don't have an extent buffer and
1883          * b) the page is already kmapped
1884          */
1885         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1886                 sblock->header_error = 1;
1887
1888         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {
1889                 sblock->header_error = 1;
1890                 sblock->generation_error = 1;
1891         }
1892
1893         if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
1894                 sblock->header_error = 1;
1895
1896         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1897                    BTRFS_UUID_SIZE))
1898                 sblock->header_error = 1;
1899
1900         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1901         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1902         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1903         index = 0;
1904         for (;;) {
1905                 u64 l = min_t(u64, len, mapped_size);
1906
1907                 crc = btrfs_csum_data(p, crc, l);
1908                 kunmap_atomic(mapped_buffer);
1909                 len -= l;
1910                 if (len == 0)
1911                         break;
1912                 index++;
1913                 BUG_ON(index >= sblock->page_count);
1914                 BUG_ON(!sblock->pagev[index]->page);
1915                 page = sblock->pagev[index]->page;
1916                 mapped_buffer = kmap_atomic(page);
1917                 mapped_size = PAGE_SIZE;
1918                 p = mapped_buffer;
1919         }
1920
1921         btrfs_csum_final(crc, calculated_csum);
1922         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1923                 sblock->checksum_error = 1;
1924
1925         return sblock->header_error || sblock->checksum_error;
1926 }
1927
1928 static int scrub_checksum_super(struct scrub_block *sblock)
1929 {
1930         struct btrfs_super_block *s;
1931         struct scrub_ctx *sctx = sblock->sctx;
1932         u8 calculated_csum[BTRFS_CSUM_SIZE];
1933         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1934         struct page *page;
1935         void *mapped_buffer;
1936         u64 mapped_size;
1937         void *p;
1938         u32 crc = ~(u32)0;
1939         int fail_gen = 0;
1940         int fail_cor = 0;
1941         u64 len;
1942         int index;
1943
1944         BUG_ON(sblock->page_count < 1);
1945         page = sblock->pagev[0]->page;
1946         mapped_buffer = kmap_atomic(page);
1947         s = (struct btrfs_super_block *)mapped_buffer;
1948         memcpy(on_disk_csum, s->csum, sctx->csum_size);
1949
1950         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
1951                 ++fail_cor;
1952
1953         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
1954                 ++fail_gen;
1955
1956         if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
1957                 ++fail_cor;
1958
1959         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
1960         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1961         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1962         index = 0;
1963         for (;;) {
1964                 u64 l = min_t(u64, len, mapped_size);
1965
1966                 crc = btrfs_csum_data(p, crc, l);
1967                 kunmap_atomic(mapped_buffer);
1968                 len -= l;
1969                 if (len == 0)
1970                         break;
1971                 index++;
1972                 BUG_ON(index >= sblock->page_count);
1973                 BUG_ON(!sblock->pagev[index]->page);
1974                 page = sblock->pagev[index]->page;
1975                 mapped_buffer = kmap_atomic(page);
1976                 mapped_size = PAGE_SIZE;
1977                 p = mapped_buffer;
1978         }
1979
1980         btrfs_csum_final(crc, calculated_csum);
1981         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1982                 ++fail_cor;
1983
1984         if (fail_cor + fail_gen) {
1985                 /*
1986                  * if we find an error in a super block, we just report it.
1987                  * They will get written with the next transaction commit
1988                  * anyway
1989                  */
1990                 spin_lock(&sctx->stat_lock);
1991                 ++sctx->stat.super_errors;
1992                 spin_unlock(&sctx->stat_lock);
1993                 if (fail_cor)
1994                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1995                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1996                 else
1997                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1998                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1999         }
2000
2001         return fail_cor + fail_gen;
2002 }
2003
2004 static void scrub_block_get(struct scrub_block *sblock)
2005 {
2006         atomic_inc(&sblock->refs);
2007 }
2008
2009 static void scrub_block_put(struct scrub_block *sblock)
2010 {
2011         if (atomic_dec_and_test(&sblock->refs)) {
2012                 int i;
2013
2014                 if (sblock->sparity)
2015                         scrub_parity_put(sblock->sparity);
2016
2017                 for (i = 0; i < sblock->page_count; i++)
2018                         scrub_page_put(sblock->pagev[i]);
2019                 kfree(sblock);
2020         }
2021 }
2022
2023 static void scrub_page_get(struct scrub_page *spage)
2024 {
2025         atomic_inc(&spage->refs);
2026 }
2027
2028 static void scrub_page_put(struct scrub_page *spage)
2029 {
2030         if (atomic_dec_and_test(&spage->refs)) {
2031                 if (spage->page)
2032                         __free_page(spage->page);
2033                 kfree(spage);
2034         }
2035 }
2036
2037 static void scrub_submit(struct scrub_ctx *sctx)
2038 {
2039         struct scrub_bio *sbio;
2040
2041         if (sctx->curr == -1)
2042                 return;
2043
2044         sbio = sctx->bios[sctx->curr];
2045         sctx->curr = -1;
2046         scrub_pending_bio_inc(sctx);
2047         btrfsic_submit_bio(sbio->bio);
2048 }
2049
2050 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2051                                     struct scrub_page *spage)
2052 {
2053         struct scrub_block *sblock = spage->sblock;
2054         struct scrub_bio *sbio;
2055         int ret;
2056
2057 again:
2058         /*
2059          * grab a fresh bio or wait for one to become available
2060          */
2061         while (sctx->curr == -1) {
2062                 spin_lock(&sctx->list_lock);
2063                 sctx->curr = sctx->first_free;
2064                 if (sctx->curr != -1) {
2065                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
2066                         sctx->bios[sctx->curr]->next_free = -1;
2067                         sctx->bios[sctx->curr]->page_count = 0;
2068                         spin_unlock(&sctx->list_lock);
2069                 } else {
2070                         spin_unlock(&sctx->list_lock);
2071                         wait_event(sctx->list_wait, sctx->first_free != -1);
2072                 }
2073         }
2074         sbio = sctx->bios[sctx->curr];
2075         if (sbio->page_count == 0) {
2076                 struct bio *bio;
2077
2078                 sbio->physical = spage->physical;
2079                 sbio->logical = spage->logical;
2080                 sbio->dev = spage->dev;
2081                 bio = sbio->bio;
2082                 if (!bio) {
2083                         bio = btrfs_io_bio_alloc(GFP_KERNEL,
2084                                         sctx->pages_per_rd_bio);
2085                         if (!bio)
2086                                 return -ENOMEM;
2087                         sbio->bio = bio;
2088                 }
2089
2090                 bio->bi_private = sbio;
2091                 bio->bi_end_io = scrub_bio_end_io;
2092                 bio->bi_bdev = sbio->dev->bdev;
2093                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2094                 bio_set_op_attrs(bio, REQ_OP_READ, 0);
2095                 sbio->err = 0;
2096         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2097                    spage->physical ||
2098                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2099                    spage->logical ||
2100                    sbio->dev != spage->dev) {
2101                 scrub_submit(sctx);
2102                 goto again;
2103         }
2104
2105         sbio->pagev[sbio->page_count] = spage;
2106         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2107         if (ret != PAGE_SIZE) {
2108                 if (sbio->page_count < 1) {
2109                         bio_put(sbio->bio);
2110                         sbio->bio = NULL;
2111                         return -EIO;
2112                 }
2113                 scrub_submit(sctx);
2114                 goto again;
2115         }
2116
2117         scrub_block_get(sblock); /* one for the page added to the bio */
2118         atomic_inc(&sblock->outstanding_pages);
2119         sbio->page_count++;
2120         if (sbio->page_count == sctx->pages_per_rd_bio)
2121                 scrub_submit(sctx);
2122
2123         return 0;
2124 }
2125
2126 static void scrub_missing_raid56_end_io(struct bio *bio)
2127 {
2128         struct scrub_block *sblock = bio->bi_private;
2129         struct btrfs_fs_info *fs_info = sblock->sctx->dev_root->fs_info;
2130
2131         if (bio->bi_error)
2132                 sblock->no_io_error_seen = 0;
2133
2134         bio_put(bio);
2135
2136         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2137 }
2138
2139 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2140 {
2141         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2142         struct scrub_ctx *sctx = sblock->sctx;
2143         u64 logical;
2144         struct btrfs_device *dev;
2145
2146         logical = sblock->pagev[0]->logical;
2147         dev = sblock->pagev[0]->dev;
2148
2149         if (sblock->no_io_error_seen)
2150                 scrub_recheck_block_checksum(sblock);
2151
2152         if (!sblock->no_io_error_seen) {
2153                 spin_lock(&sctx->stat_lock);
2154                 sctx->stat.read_errors++;
2155                 spin_unlock(&sctx->stat_lock);
2156                 btrfs_err_rl_in_rcu(sctx->dev_root->fs_info,
2157                         "IO error rebuilding logical %llu for dev %s",
2158                         logical, rcu_str_deref(dev->name));
2159         } else if (sblock->header_error || sblock->checksum_error) {
2160                 spin_lock(&sctx->stat_lock);
2161                 sctx->stat.uncorrectable_errors++;
2162                 spin_unlock(&sctx->stat_lock);
2163                 btrfs_err_rl_in_rcu(sctx->dev_root->fs_info,
2164                         "failed to rebuild valid logical %llu for dev %s",
2165                         logical, rcu_str_deref(dev->name));
2166         } else {
2167                 scrub_write_block_to_dev_replace(sblock);
2168         }
2169
2170         scrub_block_put(sblock);
2171
2172         if (sctx->is_dev_replace &&
2173             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2174                 mutex_lock(&sctx->wr_ctx.wr_lock);
2175                 scrub_wr_submit(sctx);
2176                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2177         }
2178
2179         scrub_pending_bio_dec(sctx);
2180 }
2181
2182 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2183 {
2184         struct scrub_ctx *sctx = sblock->sctx;
2185         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2186         u64 length = sblock->page_count * PAGE_SIZE;
2187         u64 logical = sblock->pagev[0]->logical;
2188         struct btrfs_bio *bbio = NULL;
2189         struct bio *bio;
2190         struct btrfs_raid_bio *rbio;
2191         int ret;
2192         int i;
2193
2194         ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical, &length,
2195                                &bbio, 0, 1);
2196         if (ret || !bbio || !bbio->raid_map)
2197                 goto bbio_out;
2198
2199         if (WARN_ON(!sctx->is_dev_replace ||
2200                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2201                 /*
2202                  * We shouldn't be scrubbing a missing device. Even for dev
2203                  * replace, we should only get here for RAID 5/6. We either
2204                  * managed to mount something with no mirrors remaining or
2205                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2206                  */
2207                 goto bbio_out;
2208         }
2209
2210         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2211         if (!bio)
2212                 goto bbio_out;
2213
2214         bio->bi_iter.bi_sector = logical >> 9;
2215         bio->bi_private = sblock;
2216         bio->bi_end_io = scrub_missing_raid56_end_io;
2217
2218         rbio = raid56_alloc_missing_rbio(sctx->dev_root, bio, bbio, length);
2219         if (!rbio)
2220                 goto rbio_out;
2221
2222         for (i = 0; i < sblock->page_count; i++) {
2223                 struct scrub_page *spage = sblock->pagev[i];
2224
2225                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2226         }
2227
2228         btrfs_init_work(&sblock->work, btrfs_scrub_helper,
2229                         scrub_missing_raid56_worker, NULL, NULL);
2230         scrub_block_get(sblock);
2231         scrub_pending_bio_inc(sctx);
2232         raid56_submit_missing_rbio(rbio);
2233         return;
2234
2235 rbio_out:
2236         bio_put(bio);
2237 bbio_out:
2238         btrfs_put_bbio(bbio);
2239         spin_lock(&sctx->stat_lock);
2240         sctx->stat.malloc_errors++;
2241         spin_unlock(&sctx->stat_lock);
2242 }
2243
2244 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2245                        u64 physical, struct btrfs_device *dev, u64 flags,
2246                        u64 gen, int mirror_num, u8 *csum, int force,
2247                        u64 physical_for_dev_replace)
2248 {
2249         struct scrub_block *sblock;
2250         int index;
2251
2252         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2253         if (!sblock) {
2254                 spin_lock(&sctx->stat_lock);
2255                 sctx->stat.malloc_errors++;
2256                 spin_unlock(&sctx->stat_lock);
2257                 return -ENOMEM;
2258         }
2259
2260         /* one ref inside this function, plus one for each page added to
2261          * a bio later on */
2262         atomic_set(&sblock->refs, 1);
2263         sblock->sctx = sctx;
2264         sblock->no_io_error_seen = 1;
2265
2266         for (index = 0; len > 0; index++) {
2267                 struct scrub_page *spage;
2268                 u64 l = min_t(u64, len, PAGE_SIZE);
2269
2270                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2271                 if (!spage) {
2272 leave_nomem:
2273                         spin_lock(&sctx->stat_lock);
2274                         sctx->stat.malloc_errors++;
2275                         spin_unlock(&sctx->stat_lock);
2276                         scrub_block_put(sblock);
2277                         return -ENOMEM;
2278                 }
2279                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2280                 scrub_page_get(spage);
2281                 sblock->pagev[index] = spage;
2282                 spage->sblock = sblock;
2283                 spage->dev = dev;
2284                 spage->flags = flags;
2285                 spage->generation = gen;
2286                 spage->logical = logical;
2287                 spage->physical = physical;
2288                 spage->physical_for_dev_replace = physical_for_dev_replace;
2289                 spage->mirror_num = mirror_num;
2290                 if (csum) {
2291                         spage->have_csum = 1;
2292                         memcpy(spage->csum, csum, sctx->csum_size);
2293                 } else {
2294                         spage->have_csum = 0;
2295                 }
2296                 sblock->page_count++;
2297                 spage->page = alloc_page(GFP_KERNEL);
2298                 if (!spage->page)
2299                         goto leave_nomem;
2300                 len -= l;
2301                 logical += l;
2302                 physical += l;
2303                 physical_for_dev_replace += l;
2304         }
2305
2306         WARN_ON(sblock->page_count == 0);
2307         if (dev->missing) {
2308                 /*
2309                  * This case should only be hit for RAID 5/6 device replace. See
2310                  * the comment in scrub_missing_raid56_pages() for details.
2311                  */
2312                 scrub_missing_raid56_pages(sblock);
2313         } else {
2314                 for (index = 0; index < sblock->page_count; index++) {
2315                         struct scrub_page *spage = sblock->pagev[index];
2316                         int ret;
2317
2318                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2319                         if (ret) {
2320                                 scrub_block_put(sblock);
2321                                 return ret;
2322                         }
2323                 }
2324
2325                 if (force)
2326                         scrub_submit(sctx);
2327         }
2328
2329         /* last one frees, either here or in bio completion for last page */
2330         scrub_block_put(sblock);
2331         return 0;
2332 }
2333
2334 static void scrub_bio_end_io(struct bio *bio)
2335 {
2336         struct scrub_bio *sbio = bio->bi_private;
2337         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2338
2339         sbio->err = bio->bi_error;
2340         sbio->bio = bio;
2341
2342         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2343 }
2344
2345 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2346 {
2347         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2348         struct scrub_ctx *sctx = sbio->sctx;
2349         int i;
2350
2351         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2352         if (sbio->err) {
2353                 for (i = 0; i < sbio->page_count; i++) {
2354                         struct scrub_page *spage = sbio->pagev[i];
2355
2356                         spage->io_error = 1;
2357                         spage->sblock->no_io_error_seen = 0;
2358                 }
2359         }
2360
2361         /* now complete the scrub_block items that have all pages completed */
2362         for (i = 0; i < sbio->page_count; i++) {
2363                 struct scrub_page *spage = sbio->pagev[i];
2364                 struct scrub_block *sblock = spage->sblock;
2365
2366                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2367                         scrub_block_complete(sblock);
2368                 scrub_block_put(sblock);
2369         }
2370
2371         bio_put(sbio->bio);
2372         sbio->bio = NULL;
2373         spin_lock(&sctx->list_lock);
2374         sbio->next_free = sctx->first_free;
2375         sctx->first_free = sbio->index;
2376         spin_unlock(&sctx->list_lock);
2377
2378         if (sctx->is_dev_replace &&
2379             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2380                 mutex_lock(&sctx->wr_ctx.wr_lock);
2381                 scrub_wr_submit(sctx);
2382                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2383         }
2384
2385         scrub_pending_bio_dec(sctx);
2386 }
2387
2388 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2389                                        unsigned long *bitmap,
2390                                        u64 start, u64 len)
2391 {
2392         u32 offset;
2393         int nsectors;
2394         int sectorsize = sparity->sctx->dev_root->sectorsize;
2395
2396         if (len >= sparity->stripe_len) {
2397                 bitmap_set(bitmap, 0, sparity->nsectors);
2398                 return;
2399         }
2400
2401         start -= sparity->logic_start;
2402         start = div_u64_rem(start, sparity->stripe_len, &offset);
2403         offset /= sectorsize;
2404         nsectors = (int)len / sectorsize;
2405
2406         if (offset + nsectors <= sparity->nsectors) {
2407                 bitmap_set(bitmap, offset, nsectors);
2408                 return;
2409         }
2410
2411         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2412         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2413 }
2414
2415 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2416                                                    u64 start, u64 len)
2417 {
2418         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2419 }
2420
2421 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2422                                                   u64 start, u64 len)
2423 {
2424         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2425 }
2426
2427 static void scrub_block_complete(struct scrub_block *sblock)
2428 {
2429         int corrupted = 0;
2430
2431         if (!sblock->no_io_error_seen) {
2432                 corrupted = 1;
2433                 scrub_handle_errored_block(sblock);
2434         } else {
2435                 /*
2436                  * if has checksum error, write via repair mechanism in
2437                  * dev replace case, otherwise write here in dev replace
2438                  * case.
2439                  */
2440                 corrupted = scrub_checksum(sblock);
2441                 if (!corrupted && sblock->sctx->is_dev_replace)
2442                         scrub_write_block_to_dev_replace(sblock);
2443         }
2444
2445         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2446                 u64 start = sblock->pagev[0]->logical;
2447                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2448                           PAGE_SIZE;
2449
2450                 scrub_parity_mark_sectors_error(sblock->sparity,
2451                                                 start, end - start);
2452         }
2453 }
2454
2455 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
2456 {
2457         struct btrfs_ordered_sum *sum = NULL;
2458         unsigned long index;
2459         unsigned long num_sectors;
2460
2461         while (!list_empty(&sctx->csum_list)) {
2462                 sum = list_first_entry(&sctx->csum_list,
2463                                        struct btrfs_ordered_sum, list);
2464                 if (sum->bytenr > logical)
2465                         return 0;
2466                 if (sum->bytenr + sum->len > logical)
2467                         break;
2468
2469                 ++sctx->stat.csum_discards;
2470                 list_del(&sum->list);
2471                 kfree(sum);
2472                 sum = NULL;
2473         }
2474         if (!sum)
2475                 return 0;
2476
2477         index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2478         num_sectors = sum->len / sctx->sectorsize;
2479         memcpy(csum, sum->sums + index, sctx->csum_size);
2480         if (index == num_sectors - 1) {
2481                 list_del(&sum->list);
2482                 kfree(sum);
2483         }
2484         return 1;
2485 }
2486
2487 /* scrub extent tries to collect up to 64 kB for each bio */
2488 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2489                         u64 physical, struct btrfs_device *dev, u64 flags,
2490                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2491 {
2492         int ret;
2493         u8 csum[BTRFS_CSUM_SIZE];
2494         u32 blocksize;
2495
2496         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2497                 blocksize = sctx->sectorsize;
2498                 spin_lock(&sctx->stat_lock);
2499                 sctx->stat.data_extents_scrubbed++;
2500                 sctx->stat.data_bytes_scrubbed += len;
2501                 spin_unlock(&sctx->stat_lock);
2502         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2503                 blocksize = sctx->nodesize;
2504                 spin_lock(&sctx->stat_lock);
2505                 sctx->stat.tree_extents_scrubbed++;
2506                 sctx->stat.tree_bytes_scrubbed += len;
2507                 spin_unlock(&sctx->stat_lock);
2508         } else {
2509                 blocksize = sctx->sectorsize;
2510                 WARN_ON(1);
2511         }
2512
2513         while (len) {
2514                 u64 l = min_t(u64, len, blocksize);
2515                 int have_csum = 0;
2516
2517                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2518                         /* push csums to sbio */
2519                         have_csum = scrub_find_csum(sctx, logical, csum);
2520                         if (have_csum == 0)
2521                                 ++sctx->stat.no_csum;
2522                         if (sctx->is_dev_replace && !have_csum) {
2523                                 ret = copy_nocow_pages(sctx, logical, l,
2524                                                        mirror_num,
2525                                                       physical_for_dev_replace);
2526                                 goto behind_scrub_pages;
2527                         }
2528                 }
2529                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2530                                   mirror_num, have_csum ? csum : NULL, 0,
2531                                   physical_for_dev_replace);
2532 behind_scrub_pages:
2533                 if (ret)
2534                         return ret;
2535                 len -= l;
2536                 logical += l;
2537                 physical += l;
2538                 physical_for_dev_replace += l;
2539         }
2540         return 0;
2541 }
2542
2543 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2544                                   u64 logical, u64 len,
2545                                   u64 physical, struct btrfs_device *dev,
2546                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2547 {
2548         struct scrub_ctx *sctx = sparity->sctx;
2549         struct scrub_block *sblock;
2550         int index;
2551
2552         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2553         if (!sblock) {
2554                 spin_lock(&sctx->stat_lock);
2555                 sctx->stat.malloc_errors++;
2556                 spin_unlock(&sctx->stat_lock);
2557                 return -ENOMEM;
2558         }
2559
2560         /* one ref inside this function, plus one for each page added to
2561          * a bio later on */
2562         atomic_set(&sblock->refs, 1);
2563         sblock->sctx = sctx;
2564         sblock->no_io_error_seen = 1;
2565         sblock->sparity = sparity;
2566         scrub_parity_get(sparity);
2567
2568         for (index = 0; len > 0; index++) {
2569                 struct scrub_page *spage;
2570                 u64 l = min_t(u64, len, PAGE_SIZE);
2571
2572                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2573                 if (!spage) {
2574 leave_nomem:
2575                         spin_lock(&sctx->stat_lock);
2576                         sctx->stat.malloc_errors++;
2577                         spin_unlock(&sctx->stat_lock);
2578                         scrub_block_put(sblock);
2579                         return -ENOMEM;
2580                 }
2581                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2582                 /* For scrub block */
2583                 scrub_page_get(spage);
2584                 sblock->pagev[index] = spage;
2585                 /* For scrub parity */
2586                 scrub_page_get(spage);
2587                 list_add_tail(&spage->list, &sparity->spages);
2588                 spage->sblock = sblock;
2589                 spage->dev = dev;
2590                 spage->flags = flags;
2591                 spage->generation = gen;
2592                 spage->logical = logical;
2593                 spage->physical = physical;
2594                 spage->mirror_num = mirror_num;
2595                 if (csum) {
2596                         spage->have_csum = 1;
2597                         memcpy(spage->csum, csum, sctx->csum_size);
2598                 } else {
2599                         spage->have_csum = 0;
2600                 }
2601                 sblock->page_count++;
2602                 spage->page = alloc_page(GFP_KERNEL);
2603                 if (!spage->page)
2604                         goto leave_nomem;
2605                 len -= l;
2606                 logical += l;
2607                 physical += l;
2608         }
2609
2610         WARN_ON(sblock->page_count == 0);
2611         for (index = 0; index < sblock->page_count; index++) {
2612                 struct scrub_page *spage = sblock->pagev[index];
2613                 int ret;
2614
2615                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2616                 if (ret) {
2617                         scrub_block_put(sblock);
2618                         return ret;
2619                 }
2620         }
2621
2622         /* last one frees, either here or in bio completion for last page */
2623         scrub_block_put(sblock);
2624         return 0;
2625 }
2626
2627 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2628                                    u64 logical, u64 len,
2629                                    u64 physical, struct btrfs_device *dev,
2630                                    u64 flags, u64 gen, int mirror_num)
2631 {
2632         struct scrub_ctx *sctx = sparity->sctx;
2633         int ret;
2634         u8 csum[BTRFS_CSUM_SIZE];
2635         u32 blocksize;
2636
2637         if (dev->missing) {
2638                 scrub_parity_mark_sectors_error(sparity, logical, len);
2639                 return 0;
2640         }
2641
2642         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2643                 blocksize = sctx->sectorsize;
2644         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2645                 blocksize = sctx->nodesize;
2646         } else {
2647                 blocksize = sctx->sectorsize;
2648                 WARN_ON(1);
2649         }
2650
2651         while (len) {
2652                 u64 l = min_t(u64, len, blocksize);
2653                 int have_csum = 0;
2654
2655                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2656                         /* push csums to sbio */
2657                         have_csum = scrub_find_csum(sctx, logical, csum);
2658                         if (have_csum == 0)
2659                                 goto skip;
2660                 }
2661                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2662                                              flags, gen, mirror_num,
2663                                              have_csum ? csum : NULL);
2664                 if (ret)
2665                         return ret;
2666 skip:
2667                 len -= l;
2668                 logical += l;
2669                 physical += l;
2670         }
2671         return 0;
2672 }
2673
2674 /*
2675  * Given a physical address, this will calculate it's
2676  * logical offset. if this is a parity stripe, it will return
2677  * the most left data stripe's logical offset.
2678  *
2679  * return 0 if it is a data stripe, 1 means parity stripe.
2680  */
2681 static int get_raid56_logic_offset(u64 physical, int num,
2682                                    struct map_lookup *map, u64 *offset,
2683                                    u64 *stripe_start)
2684 {
2685         int i;
2686         int j = 0;
2687         u64 stripe_nr;
2688         u64 last_offset;
2689         u32 stripe_index;
2690         u32 rot;
2691
2692         last_offset = (physical - map->stripes[num].physical) *
2693                       nr_data_stripes(map);
2694         if (stripe_start)
2695                 *stripe_start = last_offset;
2696
2697         *offset = last_offset;
2698         for (i = 0; i < nr_data_stripes(map); i++) {
2699                 *offset = last_offset + i * map->stripe_len;
2700
2701                 stripe_nr = div_u64(*offset, map->stripe_len);
2702                 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2703
2704                 /* Work out the disk rotation on this stripe-set */
2705                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2706                 /* calculate which stripe this data locates */
2707                 rot += i;
2708                 stripe_index = rot % map->num_stripes;
2709                 if (stripe_index == num)
2710                         return 0;
2711                 if (stripe_index < num)
2712                         j++;
2713         }
2714         *offset = last_offset + j * map->stripe_len;
2715         return 1;
2716 }
2717
2718 static void scrub_free_parity(struct scrub_parity *sparity)
2719 {
2720         struct scrub_ctx *sctx = sparity->sctx;
2721         struct scrub_page *curr, *next;
2722         int nbits;
2723
2724         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2725         if (nbits) {
2726                 spin_lock(&sctx->stat_lock);
2727                 sctx->stat.read_errors += nbits;
2728                 sctx->stat.uncorrectable_errors += nbits;
2729                 spin_unlock(&sctx->stat_lock);
2730         }
2731
2732         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2733                 list_del_init(&curr->list);
2734                 scrub_page_put(curr);
2735         }
2736
2737         kfree(sparity);
2738 }
2739
2740 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2741 {
2742         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2743                                                     work);
2744         struct scrub_ctx *sctx = sparity->sctx;
2745
2746         scrub_free_parity(sparity);
2747         scrub_pending_bio_dec(sctx);
2748 }
2749
2750 static void scrub_parity_bio_endio(struct bio *bio)
2751 {
2752         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2753
2754         if (bio->bi_error)
2755                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2756                           sparity->nsectors);
2757
2758         bio_put(bio);
2759
2760         btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
2761                         scrub_parity_bio_endio_worker, NULL, NULL);
2762         btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,
2763                          &sparity->work);
2764 }
2765
2766 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2767 {
2768         struct scrub_ctx *sctx = sparity->sctx;
2769         struct bio *bio;
2770         struct btrfs_raid_bio *rbio;
2771         struct scrub_page *spage;
2772         struct btrfs_bio *bbio = NULL;
2773         u64 length;
2774         int ret;
2775
2776         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2777                            sparity->nsectors))
2778                 goto out;
2779
2780         length = sparity->logic_end - sparity->logic_start;
2781         ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
2782                                sparity->logic_start,
2783                                &length, &bbio, 0, 1);
2784         if (ret || !bbio || !bbio->raid_map)
2785                 goto bbio_out;
2786
2787         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2788         if (!bio)
2789                 goto bbio_out;
2790
2791         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2792         bio->bi_private = sparity;
2793         bio->bi_end_io = scrub_parity_bio_endio;
2794
2795         rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
2796                                               length, sparity->scrub_dev,
2797                                               sparity->dbitmap,
2798                                               sparity->nsectors);
2799         if (!rbio)
2800                 goto rbio_out;
2801
2802         list_for_each_entry(spage, &sparity->spages, list)
2803                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2804
2805         scrub_pending_bio_inc(sctx);
2806         raid56_parity_submit_scrub_rbio(rbio);
2807         return;
2808
2809 rbio_out:
2810         bio_put(bio);
2811 bbio_out:
2812         btrfs_put_bbio(bbio);
2813         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2814                   sparity->nsectors);
2815         spin_lock(&sctx->stat_lock);
2816         sctx->stat.malloc_errors++;
2817         spin_unlock(&sctx->stat_lock);
2818 out:
2819         scrub_free_parity(sparity);
2820 }
2821
2822 static inline int scrub_calc_parity_bitmap_len(int nsectors)
2823 {
2824         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
2825 }
2826
2827 static void scrub_parity_get(struct scrub_parity *sparity)
2828 {
2829         atomic_inc(&sparity->refs);
2830 }
2831
2832 static void scrub_parity_put(struct scrub_parity *sparity)
2833 {
2834         if (!atomic_dec_and_test(&sparity->refs))
2835                 return;
2836
2837         scrub_parity_check_and_repair(sparity);
2838 }
2839
2840 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2841                                                   struct map_lookup *map,
2842                                                   struct btrfs_device *sdev,
2843                                                   struct btrfs_path *path,
2844                                                   u64 logic_start,
2845                                                   u64 logic_end)
2846 {
2847         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2848         struct btrfs_root *root = fs_info->extent_root;
2849         struct btrfs_root *csum_root = fs_info->csum_root;
2850         struct btrfs_extent_item *extent;
2851         struct btrfs_bio *bbio = NULL;
2852         u64 flags;
2853         int ret;
2854         int slot;
2855         struct extent_buffer *l;
2856         struct btrfs_key key;
2857         u64 generation;
2858         u64 extent_logical;
2859         u64 extent_physical;
2860         u64 extent_len;
2861         u64 mapped_length;
2862         struct btrfs_device *extent_dev;
2863         struct scrub_parity *sparity;
2864         int nsectors;
2865         int bitmap_len;
2866         int extent_mirror_num;
2867         int stop_loop = 0;
2868
2869         nsectors = div_u64(map->stripe_len, root->sectorsize);
2870         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2871         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2872                           GFP_NOFS);
2873         if (!sparity) {
2874                 spin_lock(&sctx->stat_lock);
2875                 sctx->stat.malloc_errors++;
2876                 spin_unlock(&sctx->stat_lock);
2877                 return -ENOMEM;
2878         }
2879
2880         sparity->stripe_len = map->stripe_len;
2881         sparity->nsectors = nsectors;
2882         sparity->sctx = sctx;
2883         sparity->scrub_dev = sdev;
2884         sparity->logic_start = logic_start;
2885         sparity->logic_end = logic_end;
2886         atomic_set(&sparity->refs, 1);
2887         INIT_LIST_HEAD(&sparity->spages);
2888         sparity->dbitmap = sparity->bitmap;
2889         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2890
2891         ret = 0;
2892         while (logic_start < logic_end) {
2893                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2894                         key.type = BTRFS_METADATA_ITEM_KEY;
2895                 else
2896                         key.type = BTRFS_EXTENT_ITEM_KEY;
2897                 key.objectid = logic_start;
2898                 key.offset = (u64)-1;
2899
2900                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2901                 if (ret < 0)
2902                         goto out;
2903
2904                 if (ret > 0) {
2905                         ret = btrfs_previous_extent_item(root, path, 0);
2906                         if (ret < 0)
2907                                 goto out;
2908                         if (ret > 0) {
2909                                 btrfs_release_path(path);
2910                                 ret = btrfs_search_slot(NULL, root, &key,
2911                                                         path, 0, 0);
2912                                 if (ret < 0)
2913                                         goto out;
2914                         }
2915                 }
2916
2917                 stop_loop = 0;
2918                 while (1) {
2919                         u64 bytes;
2920
2921                         l = path->nodes[0];
2922                         slot = path->slots[0];
2923                         if (slot >= btrfs_header_nritems(l)) {
2924                                 ret = btrfs_next_leaf(root, path);
2925                                 if (ret == 0)
2926                                         continue;
2927                                 if (ret < 0)
2928                                         goto out;
2929
2930                                 stop_loop = 1;
2931                                 break;
2932                         }
2933                         btrfs_item_key_to_cpu(l, &key, slot);
2934
2935                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2936                             key.type != BTRFS_METADATA_ITEM_KEY)
2937                                 goto next;
2938
2939                         if (key.type == BTRFS_METADATA_ITEM_KEY)
2940                                 bytes = root->nodesize;
2941                         else
2942                                 bytes = key.offset;
2943
2944                         if (key.objectid + bytes <= logic_start)
2945                                 goto next;
2946
2947                         if (key.objectid >= logic_end) {
2948                                 stop_loop = 1;
2949                                 break;
2950                         }
2951
2952                         while (key.objectid >= logic_start + map->stripe_len)
2953                                 logic_start += map->stripe_len;
2954
2955                         extent = btrfs_item_ptr(l, slot,
2956                                                 struct btrfs_extent_item);
2957                         flags = btrfs_extent_flags(l, extent);
2958                         generation = btrfs_extent_generation(l, extent);
2959
2960                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
2961                             (key.objectid < logic_start ||
2962                              key.objectid + bytes >
2963                              logic_start + map->stripe_len)) {
2964                                 btrfs_err(fs_info, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2965                                           key.objectid, logic_start);
2966                                 spin_lock(&sctx->stat_lock);
2967                                 sctx->stat.uncorrectable_errors++;
2968                                 spin_unlock(&sctx->stat_lock);
2969                                 goto next;
2970                         }
2971 again:
2972                         extent_logical = key.objectid;
2973                         extent_len = bytes;
2974
2975                         if (extent_logical < logic_start) {
2976                                 extent_len -= logic_start - extent_logical;
2977                                 extent_logical = logic_start;
2978                         }
2979
2980                         if (extent_logical + extent_len >
2981                             logic_start + map->stripe_len)
2982                                 extent_len = logic_start + map->stripe_len -
2983                                              extent_logical;
2984
2985                         scrub_parity_mark_sectors_data(sparity, extent_logical,
2986                                                        extent_len);
2987
2988                         mapped_length = extent_len;
2989                         bbio = NULL;
2990                         ret = btrfs_map_block(fs_info, READ, extent_logical,
2991                                               &mapped_length, &bbio, 0);
2992                         if (!ret) {
2993                                 if (!bbio || mapped_length < extent_len)
2994                                         ret = -EIO;
2995                         }
2996                         if (ret) {
2997                                 btrfs_put_bbio(bbio);
2998                                 goto out;
2999                         }
3000                         extent_physical = bbio->stripes[0].physical;
3001                         extent_mirror_num = bbio->mirror_num;
3002                         extent_dev = bbio->stripes[0].dev;
3003                         btrfs_put_bbio(bbio);
3004
3005                         ret = btrfs_lookup_csums_range(csum_root,
3006                                                 extent_logical,
3007                                                 extent_logical + extent_len - 1,
3008                                                 &sctx->csum_list, 1);
3009                         if (ret)
3010                                 goto out;
3011
3012                         ret = scrub_extent_for_parity(sparity, extent_logical,
3013                                                       extent_len,
3014                                                       extent_physical,
3015                                                       extent_dev, flags,
3016                                                       generation,
3017                                                       extent_mirror_num);
3018
3019                         scrub_free_csums(sctx);
3020
3021                         if (ret)
3022                                 goto out;
3023
3024                         if (extent_logical + extent_len <
3025                             key.objectid + bytes) {
3026                                 logic_start += map->stripe_len;
3027
3028                                 if (logic_start >= logic_end) {
3029                                         stop_loop = 1;
3030                                         break;
3031                                 }
3032
3033                                 if (logic_start < key.objectid + bytes) {
3034                                         cond_resched();
3035                                         goto again;
3036                                 }
3037                         }
3038 next:
3039                         path->slots[0]++;
3040                 }
3041
3042                 btrfs_release_path(path);
3043
3044                 if (stop_loop)
3045                         break;
3046
3047                 logic_start += map->stripe_len;
3048         }
3049 out:
3050         if (ret < 0)
3051                 scrub_parity_mark_sectors_error(sparity, logic_start,
3052                                                 logic_end - logic_start);
3053         scrub_parity_put(sparity);
3054         scrub_submit(sctx);
3055         mutex_lock(&sctx->wr_ctx.wr_lock);
3056         scrub_wr_submit(sctx);
3057         mutex_unlock(&sctx->wr_ctx.wr_lock);
3058
3059         btrfs_release_path(path);
3060         return ret < 0 ? ret : 0;
3061 }
3062
3063 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3064                                            struct map_lookup *map,
3065                                            struct btrfs_device *scrub_dev,
3066                                            int num, u64 base, u64 length,
3067                                            int is_dev_replace)
3068 {
3069         struct btrfs_path *path, *ppath;
3070         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3071         struct btrfs_root *root = fs_info->extent_root;
3072         struct btrfs_root *csum_root = fs_info->csum_root;
3073         struct btrfs_extent_item *extent;
3074         struct blk_plug plug;
3075         u64 flags;
3076         int ret;
3077         int slot;
3078         u64 nstripes;
3079         struct extent_buffer *l;
3080         u64 physical;
3081         u64 logical;
3082         u64 logic_end;
3083         u64 physical_end;
3084         u64 generation;
3085         int mirror_num;
3086         struct reada_control *reada1;
3087         struct reada_control *reada2;
3088         struct btrfs_key key;
3089         struct btrfs_key key_end;
3090         u64 increment = map->stripe_len;
3091         u64 offset;
3092         u64 extent_logical;
3093         u64 extent_physical;
3094         u64 extent_len;
3095         u64 stripe_logical;
3096         u64 stripe_end;
3097         struct btrfs_device *extent_dev;
3098         int extent_mirror_num;
3099         int stop_loop = 0;
3100
3101         physical = map->stripes[num].physical;
3102         offset = 0;
3103         nstripes = div_u64(length, map->stripe_len);
3104         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3105                 offset = map->stripe_len * num;
3106                 increment = map->stripe_len * map->num_stripes;
3107                 mirror_num = 1;
3108         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3109                 int factor = map->num_stripes / map->sub_stripes;
3110                 offset = map->stripe_len * (num / map->sub_stripes);
3111                 increment = map->stripe_len * factor;
3112                 mirror_num = num % map->sub_stripes + 1;
3113         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3114                 increment = map->stripe_len;
3115                 mirror_num = num % map->num_stripes + 1;
3116         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3117                 increment = map->stripe_len;
3118                 mirror_num = num % map->num_stripes + 1;
3119         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3120                 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3121                 increment = map->stripe_len * nr_data_stripes(map);
3122                 mirror_num = 1;
3123         } else {
3124                 increment = map->stripe_len;
3125                 mirror_num = 1;
3126         }
3127
3128         path = btrfs_alloc_path();
3129         if (!path)
3130                 return -ENOMEM;
3131
3132         ppath = btrfs_alloc_path();
3133         if (!ppath) {
3134                 btrfs_free_path(path);
3135                 return -ENOMEM;
3136         }
3137
3138         /*
3139          * work on commit root. The related disk blocks are static as
3140          * long as COW is applied. This means, it is save to rewrite
3141          * them to repair disk errors without any race conditions
3142          */
3143         path->search_commit_root = 1;
3144         path->skip_locking = 1;
3145
3146         ppath->search_commit_root = 1;
3147         ppath->skip_locking = 1;
3148         /*
3149          * trigger the readahead for extent tree csum tree and wait for
31