btrfs: Remove unused length var from scrub_handle_errored_block
[sfrench/cifs-2.6.git] / fs / btrfs / scrub.c
1 /*
2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public
14  * License along with this program; if not, write to the
15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16  * Boston, MA 021110-1307, USA.
17  */
18
19 #include <linux/blkdev.h>
20 #include <linux/ratelimit.h>
21 #include <linux/sched/mm.h>
22 #include "ctree.h"
23 #include "volumes.h"
24 #include "disk-io.h"
25 #include "ordered-data.h"
26 #include "transaction.h"
27 #include "backref.h"
28 #include "extent_io.h"
29 #include "dev-replace.h"
30 #include "check-integrity.h"
31 #include "rcu-string.h"
32 #include "raid56.h"
33
34 /*
35  * This is only the first step towards a full-features scrub. It reads all
36  * extent and super block and verifies the checksums. In case a bad checksum
37  * is found or the extent cannot be read, good data will be written back if
38  * any can be found.
39  *
40  * Future enhancements:
41  *  - In case an unrepairable extent is encountered, track which files are
42  *    affected and report them
43  *  - track and record media errors, throw out bad devices
44  *  - add a mode to also read unallocated space
45  */
46
47 struct scrub_block;
48 struct scrub_ctx;
49
50 /*
51  * the following three values only influence the performance.
52  * The last one configures the number of parallel and outstanding I/O
53  * operations. The first two values configure an upper limit for the number
54  * of (dynamically allocated) pages that are added to a bio.
55  */
56 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
57 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
58 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
59
60 /*
61  * the following value times PAGE_SIZE needs to be large enough to match the
62  * largest node/leaf/sector size that shall be supported.
63  * Values larger than BTRFS_STRIPE_LEN are not supported.
64  */
65 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
66
67 struct scrub_recover {
68         refcount_t              refs;
69         struct btrfs_bio        *bbio;
70         u64                     map_length;
71 };
72
73 struct scrub_page {
74         struct scrub_block      *sblock;
75         struct page             *page;
76         struct btrfs_device     *dev;
77         struct list_head        list;
78         u64                     flags;  /* extent flags */
79         u64                     generation;
80         u64                     logical;
81         u64                     physical;
82         u64                     physical_for_dev_replace;
83         atomic_t                refs;
84         struct {
85                 unsigned int    mirror_num:8;
86                 unsigned int    have_csum:1;
87                 unsigned int    io_error:1;
88         };
89         u8                      csum[BTRFS_CSUM_SIZE];
90
91         struct scrub_recover    *recover;
92 };
93
94 struct scrub_bio {
95         int                     index;
96         struct scrub_ctx        *sctx;
97         struct btrfs_device     *dev;
98         struct bio              *bio;
99         blk_status_t            status;
100         u64                     logical;
101         u64                     physical;
102 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
103         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
104 #else
105         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
106 #endif
107         int                     page_count;
108         int                     next_free;
109         struct btrfs_work       work;
110 };
111
112 struct scrub_block {
113         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
114         int                     page_count;
115         atomic_t                outstanding_pages;
116         refcount_t              refs; /* free mem on transition to zero */
117         struct scrub_ctx        *sctx;
118         struct scrub_parity     *sparity;
119         struct {
120                 unsigned int    header_error:1;
121                 unsigned int    checksum_error:1;
122                 unsigned int    no_io_error_seen:1;
123                 unsigned int    generation_error:1; /* also sets header_error */
124
125                 /* The following is for the data used to check parity */
126                 /* It is for the data with checksum */
127                 unsigned int    data_corrected:1;
128         };
129         struct btrfs_work       work;
130 };
131
132 /* Used for the chunks with parity stripe such RAID5/6 */
133 struct scrub_parity {
134         struct scrub_ctx        *sctx;
135
136         struct btrfs_device     *scrub_dev;
137
138         u64                     logic_start;
139
140         u64                     logic_end;
141
142         int                     nsectors;
143
144         u64                     stripe_len;
145
146         refcount_t              refs;
147
148         struct list_head        spages;
149
150         /* Work of parity check and repair */
151         struct btrfs_work       work;
152
153         /* Mark the parity blocks which have data */
154         unsigned long           *dbitmap;
155
156         /*
157          * Mark the parity blocks which have data, but errors happen when
158          * read data or check data
159          */
160         unsigned long           *ebitmap;
161
162         unsigned long           bitmap[0];
163 };
164
165 struct scrub_ctx {
166         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
167         struct btrfs_fs_info    *fs_info;
168         int                     first_free;
169         int                     curr;
170         atomic_t                bios_in_flight;
171         atomic_t                workers_pending;
172         spinlock_t              list_lock;
173         wait_queue_head_t       list_wait;
174         u16                     csum_size;
175         struct list_head        csum_list;
176         atomic_t                cancel_req;
177         int                     readonly;
178         int                     pages_per_rd_bio;
179
180         int                     is_dev_replace;
181
182         struct scrub_bio        *wr_curr_bio;
183         struct mutex            wr_lock;
184         int                     pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
185         struct btrfs_device     *wr_tgtdev;
186         bool                    flush_all_writes;
187
188         /*
189          * statistics
190          */
191         struct btrfs_scrub_progress stat;
192         spinlock_t              stat_lock;
193
194         /*
195          * Use a ref counter to avoid use-after-free issues. Scrub workers
196          * decrement bios_in_flight and workers_pending and then do a wakeup
197          * on the list_wait wait queue. We must ensure the main scrub task
198          * doesn't free the scrub context before or while the workers are
199          * doing the wakeup() call.
200          */
201         refcount_t              refs;
202 };
203
204 struct scrub_fixup_nodatasum {
205         struct scrub_ctx        *sctx;
206         struct btrfs_device     *dev;
207         u64                     logical;
208         struct btrfs_root       *root;
209         struct btrfs_work       work;
210         int                     mirror_num;
211 };
212
213 struct scrub_nocow_inode {
214         u64                     inum;
215         u64                     offset;
216         u64                     root;
217         struct list_head        list;
218 };
219
220 struct scrub_copy_nocow_ctx {
221         struct scrub_ctx        *sctx;
222         u64                     logical;
223         u64                     len;
224         int                     mirror_num;
225         u64                     physical_for_dev_replace;
226         struct list_head        inodes;
227         struct btrfs_work       work;
228 };
229
230 struct scrub_warning {
231         struct btrfs_path       *path;
232         u64                     extent_item_size;
233         const char              *errstr;
234         u64                     physical;
235         u64                     logical;
236         struct btrfs_device     *dev;
237 };
238
239 struct full_stripe_lock {
240         struct rb_node node;
241         u64 logical;
242         u64 refs;
243         struct mutex mutex;
244 };
245
246 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
247 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
248 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
249 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
250 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
251 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
252                                      struct scrub_block *sblocks_for_recheck);
253 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
254                                 struct scrub_block *sblock,
255                                 int retry_failed_mirror);
256 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
257 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
258                                              struct scrub_block *sblock_good);
259 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
260                                             struct scrub_block *sblock_good,
261                                             int page_num, int force_write);
262 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
263 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
264                                            int page_num);
265 static int scrub_checksum_data(struct scrub_block *sblock);
266 static int scrub_checksum_tree_block(struct scrub_block *sblock);
267 static int scrub_checksum_super(struct scrub_block *sblock);
268 static void scrub_block_get(struct scrub_block *sblock);
269 static void scrub_block_put(struct scrub_block *sblock);
270 static void scrub_page_get(struct scrub_page *spage);
271 static void scrub_page_put(struct scrub_page *spage);
272 static void scrub_parity_get(struct scrub_parity *sparity);
273 static void scrub_parity_put(struct scrub_parity *sparity);
274 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
275                                     struct scrub_page *spage);
276 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
277                        u64 physical, struct btrfs_device *dev, u64 flags,
278                        u64 gen, int mirror_num, u8 *csum, int force,
279                        u64 physical_for_dev_replace);
280 static void scrub_bio_end_io(struct bio *bio);
281 static void scrub_bio_end_io_worker(struct btrfs_work *work);
282 static void scrub_block_complete(struct scrub_block *sblock);
283 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
284                                u64 extent_logical, u64 extent_len,
285                                u64 *extent_physical,
286                                struct btrfs_device **extent_dev,
287                                int *extent_mirror_num);
288 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
289                                     struct scrub_page *spage);
290 static void scrub_wr_submit(struct scrub_ctx *sctx);
291 static void scrub_wr_bio_end_io(struct bio *bio);
292 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
293 static int write_page_nocow(struct scrub_ctx *sctx,
294                             u64 physical_for_dev_replace, struct page *page);
295 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
296                                       struct scrub_copy_nocow_ctx *ctx);
297 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
298                             int mirror_num, u64 physical_for_dev_replace);
299 static void copy_nocow_pages_worker(struct btrfs_work *work);
300 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
301 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
302 static void scrub_put_ctx(struct scrub_ctx *sctx);
303
304 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
305 {
306         return page->recover &&
307                (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
308 }
309
310 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
311 {
312         refcount_inc(&sctx->refs);
313         atomic_inc(&sctx->bios_in_flight);
314 }
315
316 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
317 {
318         atomic_dec(&sctx->bios_in_flight);
319         wake_up(&sctx->list_wait);
320         scrub_put_ctx(sctx);
321 }
322
323 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
324 {
325         while (atomic_read(&fs_info->scrub_pause_req)) {
326                 mutex_unlock(&fs_info->scrub_lock);
327                 wait_event(fs_info->scrub_pause_wait,
328                    atomic_read(&fs_info->scrub_pause_req) == 0);
329                 mutex_lock(&fs_info->scrub_lock);
330         }
331 }
332
333 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
334 {
335         atomic_inc(&fs_info->scrubs_paused);
336         wake_up(&fs_info->scrub_pause_wait);
337 }
338
339 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
340 {
341         mutex_lock(&fs_info->scrub_lock);
342         __scrub_blocked_if_needed(fs_info);
343         atomic_dec(&fs_info->scrubs_paused);
344         mutex_unlock(&fs_info->scrub_lock);
345
346         wake_up(&fs_info->scrub_pause_wait);
347 }
348
349 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
350 {
351         scrub_pause_on(fs_info);
352         scrub_pause_off(fs_info);
353 }
354
355 /*
356  * Insert new full stripe lock into full stripe locks tree
357  *
358  * Return pointer to existing or newly inserted full_stripe_lock structure if
359  * everything works well.
360  * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
361  *
362  * NOTE: caller must hold full_stripe_locks_root->lock before calling this
363  * function
364  */
365 static struct full_stripe_lock *insert_full_stripe_lock(
366                 struct btrfs_full_stripe_locks_tree *locks_root,
367                 u64 fstripe_logical)
368 {
369         struct rb_node **p;
370         struct rb_node *parent = NULL;
371         struct full_stripe_lock *entry;
372         struct full_stripe_lock *ret;
373
374         WARN_ON(!mutex_is_locked(&locks_root->lock));
375
376         p = &locks_root->root.rb_node;
377         while (*p) {
378                 parent = *p;
379                 entry = rb_entry(parent, struct full_stripe_lock, node);
380                 if (fstripe_logical < entry->logical) {
381                         p = &(*p)->rb_left;
382                 } else if (fstripe_logical > entry->logical) {
383                         p = &(*p)->rb_right;
384                 } else {
385                         entry->refs++;
386                         return entry;
387                 }
388         }
389
390         /* Insert new lock */
391         ret = kmalloc(sizeof(*ret), GFP_KERNEL);
392         if (!ret)
393                 return ERR_PTR(-ENOMEM);
394         ret->logical = fstripe_logical;
395         ret->refs = 1;
396         mutex_init(&ret->mutex);
397
398         rb_link_node(&ret->node, parent, p);
399         rb_insert_color(&ret->node, &locks_root->root);
400         return ret;
401 }
402
403 /*
404  * Search for a full stripe lock of a block group
405  *
406  * Return pointer to existing full stripe lock if found
407  * Return NULL if not found
408  */
409 static struct full_stripe_lock *search_full_stripe_lock(
410                 struct btrfs_full_stripe_locks_tree *locks_root,
411                 u64 fstripe_logical)
412 {
413         struct rb_node *node;
414         struct full_stripe_lock *entry;
415
416         WARN_ON(!mutex_is_locked(&locks_root->lock));
417
418         node = locks_root->root.rb_node;
419         while (node) {
420                 entry = rb_entry(node, struct full_stripe_lock, node);
421                 if (fstripe_logical < entry->logical)
422                         node = node->rb_left;
423                 else if (fstripe_logical > entry->logical)
424                         node = node->rb_right;
425                 else
426                         return entry;
427         }
428         return NULL;
429 }
430
431 /*
432  * Helper to get full stripe logical from a normal bytenr.
433  *
434  * Caller must ensure @cache is a RAID56 block group.
435  */
436 static u64 get_full_stripe_logical(struct btrfs_block_group_cache *cache,
437                                    u64 bytenr)
438 {
439         u64 ret;
440
441         /*
442          * Due to chunk item size limit, full stripe length should not be
443          * larger than U32_MAX. Just a sanity check here.
444          */
445         WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);
446
447         /*
448          * round_down() can only handle power of 2, while RAID56 full
449          * stripe length can be 64KiB * n, so we need to manually round down.
450          */
451         ret = div64_u64(bytenr - cache->key.objectid, cache->full_stripe_len) *
452                 cache->full_stripe_len + cache->key.objectid;
453         return ret;
454 }
455
456 /*
457  * Lock a full stripe to avoid concurrency of recovery and read
458  *
459  * It's only used for profiles with parities (RAID5/6), for other profiles it
460  * does nothing.
461  *
462  * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
463  * So caller must call unlock_full_stripe() at the same context.
464  *
465  * Return <0 if encounters error.
466  */
467 static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
468                             bool *locked_ret)
469 {
470         struct btrfs_block_group_cache *bg_cache;
471         struct btrfs_full_stripe_locks_tree *locks_root;
472         struct full_stripe_lock *existing;
473         u64 fstripe_start;
474         int ret = 0;
475
476         *locked_ret = false;
477         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
478         if (!bg_cache) {
479                 ASSERT(0);
480                 return -ENOENT;
481         }
482
483         /* Profiles not based on parity don't need full stripe lock */
484         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
485                 goto out;
486         locks_root = &bg_cache->full_stripe_locks_root;
487
488         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
489
490         /* Now insert the full stripe lock */
491         mutex_lock(&locks_root->lock);
492         existing = insert_full_stripe_lock(locks_root, fstripe_start);
493         mutex_unlock(&locks_root->lock);
494         if (IS_ERR(existing)) {
495                 ret = PTR_ERR(existing);
496                 goto out;
497         }
498         mutex_lock(&existing->mutex);
499         *locked_ret = true;
500 out:
501         btrfs_put_block_group(bg_cache);
502         return ret;
503 }
504
505 /*
506  * Unlock a full stripe.
507  *
508  * NOTE: Caller must ensure it's the same context calling corresponding
509  * lock_full_stripe().
510  *
511  * Return 0 if we unlock full stripe without problem.
512  * Return <0 for error
513  */
514 static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
515                               bool locked)
516 {
517         struct btrfs_block_group_cache *bg_cache;
518         struct btrfs_full_stripe_locks_tree *locks_root;
519         struct full_stripe_lock *fstripe_lock;
520         u64 fstripe_start;
521         bool freeit = false;
522         int ret = 0;
523
524         /* If we didn't acquire full stripe lock, no need to continue */
525         if (!locked)
526                 return 0;
527
528         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
529         if (!bg_cache) {
530                 ASSERT(0);
531                 return -ENOENT;
532         }
533         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
534                 goto out;
535
536         locks_root = &bg_cache->full_stripe_locks_root;
537         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
538
539         mutex_lock(&locks_root->lock);
540         fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
541         /* Unpaired unlock_full_stripe() detected */
542         if (!fstripe_lock) {
543                 WARN_ON(1);
544                 ret = -ENOENT;
545                 mutex_unlock(&locks_root->lock);
546                 goto out;
547         }
548
549         if (fstripe_lock->refs == 0) {
550                 WARN_ON(1);
551                 btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
552                         fstripe_lock->logical);
553         } else {
554                 fstripe_lock->refs--;
555         }
556
557         if (fstripe_lock->refs == 0) {
558                 rb_erase(&fstripe_lock->node, &locks_root->root);
559                 freeit = true;
560         }
561         mutex_unlock(&locks_root->lock);
562
563         mutex_unlock(&fstripe_lock->mutex);
564         if (freeit)
565                 kfree(fstripe_lock);
566 out:
567         btrfs_put_block_group(bg_cache);
568         return ret;
569 }
570
571 /*
572  * used for workers that require transaction commits (i.e., for the
573  * NOCOW case)
574  */
575 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
576 {
577         struct btrfs_fs_info *fs_info = sctx->fs_info;
578
579         refcount_inc(&sctx->refs);
580         /*
581          * increment scrubs_running to prevent cancel requests from
582          * completing as long as a worker is running. we must also
583          * increment scrubs_paused to prevent deadlocking on pause
584          * requests used for transactions commits (as the worker uses a
585          * transaction context). it is safe to regard the worker
586          * as paused for all matters practical. effectively, we only
587          * avoid cancellation requests from completing.
588          */
589         mutex_lock(&fs_info->scrub_lock);
590         atomic_inc(&fs_info->scrubs_running);
591         atomic_inc(&fs_info->scrubs_paused);
592         mutex_unlock(&fs_info->scrub_lock);
593
594         /*
595          * check if @scrubs_running=@scrubs_paused condition
596          * inside wait_event() is not an atomic operation.
597          * which means we may inc/dec @scrub_running/paused
598          * at any time. Let's wake up @scrub_pause_wait as
599          * much as we can to let commit transaction blocked less.
600          */
601         wake_up(&fs_info->scrub_pause_wait);
602
603         atomic_inc(&sctx->workers_pending);
604 }
605
606 /* used for workers that require transaction commits */
607 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
608 {
609         struct btrfs_fs_info *fs_info = sctx->fs_info;
610
611         /*
612          * see scrub_pending_trans_workers_inc() why we're pretending
613          * to be paused in the scrub counters
614          */
615         mutex_lock(&fs_info->scrub_lock);
616         atomic_dec(&fs_info->scrubs_running);
617         atomic_dec(&fs_info->scrubs_paused);
618         mutex_unlock(&fs_info->scrub_lock);
619         atomic_dec(&sctx->workers_pending);
620         wake_up(&fs_info->scrub_pause_wait);
621         wake_up(&sctx->list_wait);
622         scrub_put_ctx(sctx);
623 }
624
625 static void scrub_free_csums(struct scrub_ctx *sctx)
626 {
627         while (!list_empty(&sctx->csum_list)) {
628                 struct btrfs_ordered_sum *sum;
629                 sum = list_first_entry(&sctx->csum_list,
630                                        struct btrfs_ordered_sum, list);
631                 list_del(&sum->list);
632                 kfree(sum);
633         }
634 }
635
636 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
637 {
638         int i;
639
640         if (!sctx)
641                 return;
642
643         /* this can happen when scrub is cancelled */
644         if (sctx->curr != -1) {
645                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
646
647                 for (i = 0; i < sbio->page_count; i++) {
648                         WARN_ON(!sbio->pagev[i]->page);
649                         scrub_block_put(sbio->pagev[i]->sblock);
650                 }
651                 bio_put(sbio->bio);
652         }
653
654         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
655                 struct scrub_bio *sbio = sctx->bios[i];
656
657                 if (!sbio)
658                         break;
659                 kfree(sbio);
660         }
661
662         kfree(sctx->wr_curr_bio);
663         scrub_free_csums(sctx);
664         kfree(sctx);
665 }
666
667 static void scrub_put_ctx(struct scrub_ctx *sctx)
668 {
669         if (refcount_dec_and_test(&sctx->refs))
670                 scrub_free_ctx(sctx);
671 }
672
673 static noinline_for_stack
674 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
675 {
676         struct scrub_ctx *sctx;
677         int             i;
678         struct btrfs_fs_info *fs_info = dev->fs_info;
679
680         sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
681         if (!sctx)
682                 goto nomem;
683         refcount_set(&sctx->refs, 1);
684         sctx->is_dev_replace = is_dev_replace;
685         sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
686         sctx->curr = -1;
687         sctx->fs_info = dev->fs_info;
688         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
689                 struct scrub_bio *sbio;
690
691                 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
692                 if (!sbio)
693                         goto nomem;
694                 sctx->bios[i] = sbio;
695
696                 sbio->index = i;
697                 sbio->sctx = sctx;
698                 sbio->page_count = 0;
699                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
700                                 scrub_bio_end_io_worker, NULL, NULL);
701
702                 if (i != SCRUB_BIOS_PER_SCTX - 1)
703                         sctx->bios[i]->next_free = i + 1;
704                 else
705                         sctx->bios[i]->next_free = -1;
706         }
707         sctx->first_free = 0;
708         atomic_set(&sctx->bios_in_flight, 0);
709         atomic_set(&sctx->workers_pending, 0);
710         atomic_set(&sctx->cancel_req, 0);
711         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
712         INIT_LIST_HEAD(&sctx->csum_list);
713
714         spin_lock_init(&sctx->list_lock);
715         spin_lock_init(&sctx->stat_lock);
716         init_waitqueue_head(&sctx->list_wait);
717
718         WARN_ON(sctx->wr_curr_bio != NULL);
719         mutex_init(&sctx->wr_lock);
720         sctx->wr_curr_bio = NULL;
721         if (is_dev_replace) {
722                 WARN_ON(!fs_info->dev_replace.tgtdev);
723                 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
724                 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
725                 sctx->flush_all_writes = false;
726         }
727
728         return sctx;
729
730 nomem:
731         scrub_free_ctx(sctx);
732         return ERR_PTR(-ENOMEM);
733 }
734
735 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
736                                      void *warn_ctx)
737 {
738         u64 isize;
739         u32 nlink;
740         int ret;
741         int i;
742         unsigned nofs_flag;
743         struct extent_buffer *eb;
744         struct btrfs_inode_item *inode_item;
745         struct scrub_warning *swarn = warn_ctx;
746         struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
747         struct inode_fs_paths *ipath = NULL;
748         struct btrfs_root *local_root;
749         struct btrfs_key root_key;
750         struct btrfs_key key;
751
752         root_key.objectid = root;
753         root_key.type = BTRFS_ROOT_ITEM_KEY;
754         root_key.offset = (u64)-1;
755         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
756         if (IS_ERR(local_root)) {
757                 ret = PTR_ERR(local_root);
758                 goto err;
759         }
760
761         /*
762          * this makes the path point to (inum INODE_ITEM ioff)
763          */
764         key.objectid = inum;
765         key.type = BTRFS_INODE_ITEM_KEY;
766         key.offset = 0;
767
768         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
769         if (ret) {
770                 btrfs_release_path(swarn->path);
771                 goto err;
772         }
773
774         eb = swarn->path->nodes[0];
775         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
776                                         struct btrfs_inode_item);
777         isize = btrfs_inode_size(eb, inode_item);
778         nlink = btrfs_inode_nlink(eb, inode_item);
779         btrfs_release_path(swarn->path);
780
781         /*
782          * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
783          * uses GFP_NOFS in this context, so we keep it consistent but it does
784          * not seem to be strictly necessary.
785          */
786         nofs_flag = memalloc_nofs_save();
787         ipath = init_ipath(4096, local_root, swarn->path);
788         memalloc_nofs_restore(nofs_flag);
789         if (IS_ERR(ipath)) {
790                 ret = PTR_ERR(ipath);
791                 ipath = NULL;
792                 goto err;
793         }
794         ret = paths_from_inode(inum, ipath);
795
796         if (ret < 0)
797                 goto err;
798
799         /*
800          * we deliberately ignore the bit ipath might have been too small to
801          * hold all of the paths here
802          */
803         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
804                 btrfs_warn_in_rcu(fs_info,
805 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
806                                   swarn->errstr, swarn->logical,
807                                   rcu_str_deref(swarn->dev->name),
808                                   swarn->physical,
809                                   root, inum, offset,
810                                   min(isize - offset, (u64)PAGE_SIZE), nlink,
811                                   (char *)(unsigned long)ipath->fspath->val[i]);
812
813         free_ipath(ipath);
814         return 0;
815
816 err:
817         btrfs_warn_in_rcu(fs_info,
818                           "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
819                           swarn->errstr, swarn->logical,
820                           rcu_str_deref(swarn->dev->name),
821                           swarn->physical,
822                           root, inum, offset, ret);
823
824         free_ipath(ipath);
825         return 0;
826 }
827
828 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
829 {
830         struct btrfs_device *dev;
831         struct btrfs_fs_info *fs_info;
832         struct btrfs_path *path;
833         struct btrfs_key found_key;
834         struct extent_buffer *eb;
835         struct btrfs_extent_item *ei;
836         struct scrub_warning swarn;
837         unsigned long ptr = 0;
838         u64 extent_item_pos;
839         u64 flags = 0;
840         u64 ref_root;
841         u32 item_size;
842         u8 ref_level = 0;
843         int ret;
844
845         WARN_ON(sblock->page_count < 1);
846         dev = sblock->pagev[0]->dev;
847         fs_info = sblock->sctx->fs_info;
848
849         path = btrfs_alloc_path();
850         if (!path)
851                 return;
852
853         swarn.physical = sblock->pagev[0]->physical;
854         swarn.logical = sblock->pagev[0]->logical;
855         swarn.errstr = errstr;
856         swarn.dev = NULL;
857
858         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
859                                   &flags);
860         if (ret < 0)
861                 goto out;
862
863         extent_item_pos = swarn.logical - found_key.objectid;
864         swarn.extent_item_size = found_key.offset;
865
866         eb = path->nodes[0];
867         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
868         item_size = btrfs_item_size_nr(eb, path->slots[0]);
869
870         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
871                 do {
872                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
873                                                       item_size, &ref_root,
874                                                       &ref_level);
875                         btrfs_warn_in_rcu(fs_info,
876 "%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
877                                 errstr, swarn.logical,
878                                 rcu_str_deref(dev->name),
879                                 swarn.physical,
880                                 ref_level ? "node" : "leaf",
881                                 ret < 0 ? -1 : ref_level,
882                                 ret < 0 ? -1 : ref_root);
883                 } while (ret != 1);
884                 btrfs_release_path(path);
885         } else {
886                 btrfs_release_path(path);
887                 swarn.path = path;
888                 swarn.dev = dev;
889                 iterate_extent_inodes(fs_info, found_key.objectid,
890                                         extent_item_pos, 1,
891                                         scrub_print_warning_inode, &swarn, false);
892         }
893
894 out:
895         btrfs_free_path(path);
896 }
897
898 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
899 {
900         struct page *page = NULL;
901         unsigned long index;
902         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
903         int ret;
904         int corrected = 0;
905         struct btrfs_key key;
906         struct inode *inode = NULL;
907         struct btrfs_fs_info *fs_info;
908         u64 end = offset + PAGE_SIZE - 1;
909         struct btrfs_root *local_root;
910         int srcu_index;
911
912         key.objectid = root;
913         key.type = BTRFS_ROOT_ITEM_KEY;
914         key.offset = (u64)-1;
915
916         fs_info = fixup->root->fs_info;
917         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
918
919         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
920         if (IS_ERR(local_root)) {
921                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
922                 return PTR_ERR(local_root);
923         }
924
925         key.type = BTRFS_INODE_ITEM_KEY;
926         key.objectid = inum;
927         key.offset = 0;
928         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
929         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
930         if (IS_ERR(inode))
931                 return PTR_ERR(inode);
932
933         index = offset >> PAGE_SHIFT;
934
935         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
936         if (!page) {
937                 ret = -ENOMEM;
938                 goto out;
939         }
940
941         if (PageUptodate(page)) {
942                 if (PageDirty(page)) {
943                         /*
944                          * we need to write the data to the defect sector. the
945                          * data that was in that sector is not in memory,
946                          * because the page was modified. we must not write the
947                          * modified page to that sector.
948                          *
949                          * TODO: what could be done here: wait for the delalloc
950                          *       runner to write out that page (might involve
951                          *       COW) and see whether the sector is still
952                          *       referenced afterwards.
953                          *
954                          * For the meantime, we'll treat this error
955                          * incorrectable, although there is a chance that a
956                          * later scrub will find the bad sector again and that
957                          * there's no dirty page in memory, then.
958                          */
959                         ret = -EIO;
960                         goto out;
961                 }
962                 ret = repair_io_failure(fs_info, inum, offset, PAGE_SIZE,
963                                         fixup->logical, page,
964                                         offset - page_offset(page),
965                                         fixup->mirror_num);
966                 unlock_page(page);
967                 corrected = !ret;
968         } else {
969                 /*
970                  * we need to get good data first. the general readpage path
971                  * will call repair_io_failure for us, we just have to make
972                  * sure we read the bad mirror.
973                  */
974                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
975                                         EXTENT_DAMAGED);
976                 if (ret) {
977                         /* set_extent_bits should give proper error */
978                         WARN_ON(ret > 0);
979                         if (ret > 0)
980                                 ret = -EFAULT;
981                         goto out;
982                 }
983
984                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
985                                                 btrfs_get_extent,
986                                                 fixup->mirror_num);
987                 wait_on_page_locked(page);
988
989                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
990                                                 end, EXTENT_DAMAGED, 0, NULL);
991                 if (!corrected)
992                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
993                                                 EXTENT_DAMAGED);
994         }
995
996 out:
997         if (page)
998                 put_page(page);
999
1000         iput(inode);
1001
1002         if (ret < 0)
1003                 return ret;
1004
1005         if (ret == 0 && corrected) {
1006                 /*
1007                  * we only need to call readpage for one of the inodes belonging
1008                  * to this extent. so make iterate_extent_inodes stop
1009                  */
1010                 return 1;
1011         }
1012
1013         return -EIO;
1014 }
1015
1016 static void scrub_fixup_nodatasum(struct btrfs_work *work)
1017 {
1018         struct btrfs_fs_info *fs_info;
1019         int ret;
1020         struct scrub_fixup_nodatasum *fixup;
1021         struct scrub_ctx *sctx;
1022         struct btrfs_trans_handle *trans = NULL;
1023         struct btrfs_path *path;
1024         int uncorrectable = 0;
1025
1026         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
1027         sctx = fixup->sctx;
1028         fs_info = fixup->root->fs_info;
1029
1030         path = btrfs_alloc_path();
1031         if (!path) {
1032                 spin_lock(&sctx->stat_lock);
1033                 ++sctx->stat.malloc_errors;
1034                 spin_unlock(&sctx->stat_lock);
1035                 uncorrectable = 1;
1036                 goto out;
1037         }
1038
1039         trans = btrfs_join_transaction(fixup->root);
1040         if (IS_ERR(trans)) {
1041                 uncorrectable = 1;
1042                 goto out;
1043         }
1044
1045         /*
1046          * the idea is to trigger a regular read through the standard path. we
1047          * read a page from the (failed) logical address by specifying the
1048          * corresponding copynum of the failed sector. thus, that readpage is
1049          * expected to fail.
1050          * that is the point where on-the-fly error correction will kick in
1051          * (once it's finished) and rewrite the failed sector if a good copy
1052          * can be found.
1053          */
1054         ret = iterate_inodes_from_logical(fixup->logical, fs_info, path,
1055                                           scrub_fixup_readpage, fixup, false);
1056         if (ret < 0) {
1057                 uncorrectable = 1;
1058                 goto out;
1059         }
1060         WARN_ON(ret != 1);
1061
1062         spin_lock(&sctx->stat_lock);
1063         ++sctx->stat.corrected_errors;
1064         spin_unlock(&sctx->stat_lock);
1065
1066 out:
1067         if (trans && !IS_ERR(trans))
1068                 btrfs_end_transaction(trans);
1069         if (uncorrectable) {
1070                 spin_lock(&sctx->stat_lock);
1071                 ++sctx->stat.uncorrectable_errors;
1072                 spin_unlock(&sctx->stat_lock);
1073                 btrfs_dev_replace_stats_inc(
1074                         &fs_info->dev_replace.num_uncorrectable_read_errors);
1075                 btrfs_err_rl_in_rcu(fs_info,
1076                     "unable to fixup (nodatasum) error at logical %llu on dev %s",
1077                         fixup->logical, rcu_str_deref(fixup->dev->name));
1078         }
1079
1080         btrfs_free_path(path);
1081         kfree(fixup);
1082
1083         scrub_pending_trans_workers_dec(sctx);
1084 }
1085
1086 static inline void scrub_get_recover(struct scrub_recover *recover)
1087 {
1088         refcount_inc(&recover->refs);
1089 }
1090
1091 static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
1092                                      struct scrub_recover *recover)
1093 {
1094         if (refcount_dec_and_test(&recover->refs)) {
1095                 btrfs_bio_counter_dec(fs_info);
1096                 btrfs_put_bbio(recover->bbio);
1097                 kfree(recover);
1098         }
1099 }
1100
1101 /*
1102  * scrub_handle_errored_block gets called when either verification of the
1103  * pages failed or the bio failed to read, e.g. with EIO. In the latter
1104  * case, this function handles all pages in the bio, even though only one
1105  * may be bad.
1106  * The goal of this function is to repair the errored block by using the
1107  * contents of one of the mirrors.
1108  */
1109 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
1110 {
1111         struct scrub_ctx *sctx = sblock_to_check->sctx;
1112         struct btrfs_device *dev;
1113         struct btrfs_fs_info *fs_info;
1114         u64 logical;
1115         unsigned int failed_mirror_index;
1116         unsigned int is_metadata;
1117         unsigned int have_csum;
1118         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
1119         struct scrub_block *sblock_bad;
1120         int ret;
1121         int mirror_index;
1122         int page_num;
1123         int success;
1124         bool full_stripe_locked;
1125         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
1126                                       DEFAULT_RATELIMIT_BURST);
1127
1128         BUG_ON(sblock_to_check->page_count < 1);
1129         fs_info = sctx->fs_info;
1130         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
1131                 /*
1132                  * if we find an error in a super block, we just report it.
1133                  * They will get written with the next transaction commit
1134                  * anyway
1135                  */
1136                 spin_lock(&sctx->stat_lock);
1137                 ++sctx->stat.super_errors;
1138                 spin_unlock(&sctx->stat_lock);
1139                 return 0;
1140         }
1141         logical = sblock_to_check->pagev[0]->logical;
1142         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
1143         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
1144         is_metadata = !(sblock_to_check->pagev[0]->flags &
1145                         BTRFS_EXTENT_FLAG_DATA);
1146         have_csum = sblock_to_check->pagev[0]->have_csum;
1147         dev = sblock_to_check->pagev[0]->dev;
1148
1149         /*
1150          * For RAID5/6, race can happen for a different device scrub thread.
1151          * For data corruption, Parity and Data threads will both try
1152          * to recovery the data.
1153          * Race can lead to doubly added csum error, or even unrecoverable
1154          * error.
1155          */
1156         ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
1157         if (ret < 0) {
1158                 spin_lock(&sctx->stat_lock);
1159                 if (ret == -ENOMEM)
1160                         sctx->stat.malloc_errors++;
1161                 sctx->stat.read_errors++;
1162                 sctx->stat.uncorrectable_errors++;
1163                 spin_unlock(&sctx->stat_lock);
1164                 return ret;
1165         }
1166
1167         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
1168                 sblocks_for_recheck = NULL;
1169                 goto nodatasum_case;
1170         }
1171
1172         /*
1173          * read all mirrors one after the other. This includes to
1174          * re-read the extent or metadata block that failed (that was
1175          * the cause that this fixup code is called) another time,
1176          * page by page this time in order to know which pages
1177          * caused I/O errors and which ones are good (for all mirrors).
1178          * It is the goal to handle the situation when more than one
1179          * mirror contains I/O errors, but the errors do not
1180          * overlap, i.e. the data can be repaired by selecting the
1181          * pages from those mirrors without I/O error on the
1182          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
1183          * would be that mirror #1 has an I/O error on the first page,
1184          * the second page is good, and mirror #2 has an I/O error on
1185          * the second page, but the first page is good.
1186          * Then the first page of the first mirror can be repaired by
1187          * taking the first page of the second mirror, and the
1188          * second page of the second mirror can be repaired by
1189          * copying the contents of the 2nd page of the 1st mirror.
1190          * One more note: if the pages of one mirror contain I/O
1191          * errors, the checksum cannot be verified. In order to get
1192          * the best data for repairing, the first attempt is to find
1193          * a mirror without I/O errors and with a validated checksum.
1194          * Only if this is not possible, the pages are picked from
1195          * mirrors with I/O errors without considering the checksum.
1196          * If the latter is the case, at the end, the checksum of the
1197          * repaired area is verified in order to correctly maintain
1198          * the statistics.
1199          */
1200
1201         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
1202                                       sizeof(*sblocks_for_recheck), GFP_NOFS);
1203         if (!sblocks_for_recheck) {
1204                 spin_lock(&sctx->stat_lock);
1205                 sctx->stat.malloc_errors++;
1206                 sctx->stat.read_errors++;
1207                 sctx->stat.uncorrectable_errors++;
1208                 spin_unlock(&sctx->stat_lock);
1209                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1210                 goto out;
1211         }
1212
1213         /* setup the context, map the logical blocks and alloc the pages */
1214         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
1215         if (ret) {
1216                 spin_lock(&sctx->stat_lock);
1217                 sctx->stat.read_errors++;
1218                 sctx->stat.uncorrectable_errors++;
1219                 spin_unlock(&sctx->stat_lock);
1220                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1221                 goto out;
1222         }
1223         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
1224         sblock_bad = sblocks_for_recheck + failed_mirror_index;
1225
1226         /* build and submit the bios for the failed mirror, check checksums */
1227         scrub_recheck_block(fs_info, sblock_bad, 1);
1228
1229         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
1230             sblock_bad->no_io_error_seen) {
1231                 /*
1232                  * the error disappeared after reading page by page, or
1233                  * the area was part of a huge bio and other parts of the
1234                  * bio caused I/O errors, or the block layer merged several
1235                  * read requests into one and the error is caused by a
1236                  * different bio (usually one of the two latter cases is
1237                  * the cause)
1238                  */
1239                 spin_lock(&sctx->stat_lock);
1240                 sctx->stat.unverified_errors++;
1241                 sblock_to_check->data_corrected = 1;
1242                 spin_unlock(&sctx->stat_lock);
1243
1244                 if (sctx->is_dev_replace)
1245                         scrub_write_block_to_dev_replace(sblock_bad);
1246                 goto out;
1247         }
1248
1249         if (!sblock_bad->no_io_error_seen) {
1250                 spin_lock(&sctx->stat_lock);
1251                 sctx->stat.read_errors++;
1252                 spin_unlock(&sctx->stat_lock);
1253                 if (__ratelimit(&_rs))
1254                         scrub_print_warning("i/o error", sblock_to_check);
1255                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1256         } else if (sblock_bad->checksum_error) {
1257                 spin_lock(&sctx->stat_lock);
1258                 sctx->stat.csum_errors++;
1259                 spin_unlock(&sctx->stat_lock);
1260                 if (__ratelimit(&_rs))
1261                         scrub_print_warning("checksum error", sblock_to_check);
1262                 btrfs_dev_stat_inc_and_print(dev,
1263                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
1264         } else if (sblock_bad->header_error) {
1265                 spin_lock(&sctx->stat_lock);
1266                 sctx->stat.verify_errors++;
1267                 spin_unlock(&sctx->stat_lock);
1268                 if (__ratelimit(&_rs))
1269                         scrub_print_warning("checksum/header error",
1270                                             sblock_to_check);
1271                 if (sblock_bad->generation_error)
1272                         btrfs_dev_stat_inc_and_print(dev,
1273                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1274                 else
1275                         btrfs_dev_stat_inc_and_print(dev,
1276                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1277         }
1278
1279         if (sctx->readonly) {
1280                 ASSERT(!sctx->is_dev_replace);
1281                 goto out;
1282         }
1283
1284         if (!is_metadata && !have_csum) {
1285                 struct scrub_fixup_nodatasum *fixup_nodatasum;
1286
1287                 WARN_ON(sctx->is_dev_replace);
1288
1289 nodatasum_case:
1290
1291                 /*
1292                  * !is_metadata and !have_csum, this means that the data
1293                  * might not be COWed, that it might be modified
1294                  * concurrently. The general strategy to work on the
1295                  * commit root does not help in the case when COW is not
1296                  * used.
1297                  */
1298                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1299                 if (!fixup_nodatasum)
1300                         goto did_not_correct_error;
1301                 fixup_nodatasum->sctx = sctx;
1302                 fixup_nodatasum->dev = dev;
1303                 fixup_nodatasum->logical = logical;
1304                 fixup_nodatasum->root = fs_info->extent_root;
1305                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1306                 scrub_pending_trans_workers_inc(sctx);
1307                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1308                                 scrub_fixup_nodatasum, NULL, NULL);
1309                 btrfs_queue_work(fs_info->scrub_workers,
1310                                  &fixup_nodatasum->work);
1311                 goto out;
1312         }
1313
1314         /*
1315          * now build and submit the bios for the other mirrors, check
1316          * checksums.
1317          * First try to pick the mirror which is completely without I/O
1318          * errors and also does not have a checksum error.
1319          * If one is found, and if a checksum is present, the full block
1320          * that is known to contain an error is rewritten. Afterwards
1321          * the block is known to be corrected.
1322          * If a mirror is found which is completely correct, and no
1323          * checksum is present, only those pages are rewritten that had
1324          * an I/O error in the block to be repaired, since it cannot be
1325          * determined, which copy of the other pages is better (and it
1326          * could happen otherwise that a correct page would be
1327          * overwritten by a bad one).
1328          */
1329         for (mirror_index = 0; ;mirror_index++) {
1330                 struct scrub_block *sblock_other;
1331
1332                 if (mirror_index == failed_mirror_index)
1333                         continue;
1334
1335                 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
1336                 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1337                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1338                                 break;
1339                         if (!sblocks_for_recheck[mirror_index].page_count)
1340                                 break;
1341
1342                         sblock_other = sblocks_for_recheck + mirror_index;
1343                 } else {
1344                         struct scrub_recover *r = sblock_bad->pagev[0]->recover;
1345                         int max_allowed = r->bbio->num_stripes -
1346                                                 r->bbio->num_tgtdevs;
1347
1348                         if (mirror_index >= max_allowed)
1349                                 break;
1350                         if (!sblocks_for_recheck[1].page_count)
1351                                 break;
1352
1353                         ASSERT(failed_mirror_index == 0);
1354                         sblock_other = sblocks_for_recheck + 1;
1355                         sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
1356                 }
1357
1358                 /* build and submit the bios, check checksums */
1359                 scrub_recheck_block(fs_info, sblock_other, 0);
1360
1361                 if (!sblock_other->header_error &&
1362                     !sblock_other->checksum_error &&
1363                     sblock_other->no_io_error_seen) {
1364                         if (sctx->is_dev_replace) {
1365                                 scrub_write_block_to_dev_replace(sblock_other);
1366                                 goto corrected_error;
1367                         } else {
1368                                 ret = scrub_repair_block_from_good_copy(
1369                                                 sblock_bad, sblock_other);
1370                                 if (!ret)
1371                                         goto corrected_error;
1372                         }
1373                 }
1374         }
1375
1376         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1377                 goto did_not_correct_error;
1378
1379         /*
1380          * In case of I/O errors in the area that is supposed to be
1381          * repaired, continue by picking good copies of those pages.
1382          * Select the good pages from mirrors to rewrite bad pages from
1383          * the area to fix. Afterwards verify the checksum of the block
1384          * that is supposed to be repaired. This verification step is
1385          * only done for the purpose of statistic counting and for the
1386          * final scrub report, whether errors remain.
1387          * A perfect algorithm could make use of the checksum and try
1388          * all possible combinations of pages from the different mirrors
1389          * until the checksum verification succeeds. For example, when
1390          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1391          * of mirror #2 is readable but the final checksum test fails,
1392          * then the 2nd page of mirror #3 could be tried, whether now
1393          * the final checksum succeeds. But this would be a rare
1394          * exception and is therefore not implemented. At least it is
1395          * avoided that the good copy is overwritten.
1396          * A more useful improvement would be to pick the sectors
1397          * without I/O error based on sector sizes (512 bytes on legacy
1398          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1399          * mirror could be repaired by taking 512 byte of a different
1400          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1401          * area are unreadable.
1402          */
1403         success = 1;
1404         for (page_num = 0; page_num < sblock_bad->page_count;
1405              page_num++) {
1406                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1407                 struct scrub_block *sblock_other = NULL;
1408
1409                 /* skip no-io-error page in scrub */
1410                 if (!page_bad->io_error && !sctx->is_dev_replace)
1411                         continue;
1412
1413                 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1414                         /*
1415                          * In case of dev replace, if raid56 rebuild process
1416                          * didn't work out correct data, then copy the content
1417                          * in sblock_bad to make sure target device is identical
1418                          * to source device, instead of writing garbage data in
1419                          * sblock_for_recheck array to target device.
1420                          */
1421                         sblock_other = NULL;
1422                 } else if (page_bad->io_error) {
1423                         /* try to find no-io-error page in mirrors */
1424                         for (mirror_index = 0;
1425                              mirror_index < BTRFS_MAX_MIRRORS &&
1426                              sblocks_for_recheck[mirror_index].page_count > 0;
1427                              mirror_index++) {
1428                                 if (!sblocks_for_recheck[mirror_index].
1429                                     pagev[page_num]->io_error) {
1430                                         sblock_other = sblocks_for_recheck +
1431                                                        mirror_index;
1432                                         break;
1433                                 }
1434                         }
1435                         if (!sblock_other)
1436                                 success = 0;
1437                 }
1438
1439                 if (sctx->is_dev_replace) {
1440                         /*
1441                          * did not find a mirror to fetch the page
1442                          * from. scrub_write_page_to_dev_replace()
1443                          * handles this case (page->io_error), by
1444                          * filling the block with zeros before
1445                          * submitting the write request
1446                          */
1447                         if (!sblock_other)
1448                                 sblock_other = sblock_bad;
1449
1450                         if (scrub_write_page_to_dev_replace(sblock_other,
1451                                                             page_num) != 0) {
1452                                 btrfs_dev_replace_stats_inc(
1453                                         &fs_info->dev_replace.num_write_errors);
1454                                 success = 0;
1455                         }
1456                 } else if (sblock_other) {
1457                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1458                                                                sblock_other,
1459                                                                page_num, 0);
1460                         if (0 == ret)
1461                                 page_bad->io_error = 0;
1462                         else
1463                                 success = 0;
1464                 }
1465         }
1466
1467         if (success && !sctx->is_dev_replace) {
1468                 if (is_metadata || have_csum) {
1469                         /*
1470                          * need to verify the checksum now that all
1471                          * sectors on disk are repaired (the write
1472                          * request for data to be repaired is on its way).
1473                          * Just be lazy and use scrub_recheck_block()
1474                          * which re-reads the data before the checksum
1475                          * is verified, but most likely the data comes out
1476                          * of the page cache.
1477                          */
1478                         scrub_recheck_block(fs_info, sblock_bad, 1);
1479                         if (!sblock_bad->header_error &&
1480                             !sblock_bad->checksum_error &&
1481                             sblock_bad->no_io_error_seen)
1482                                 goto corrected_error;
1483                         else
1484                                 goto did_not_correct_error;
1485                 } else {
1486 corrected_error:
1487                         spin_lock(&sctx->stat_lock);
1488                         sctx->stat.corrected_errors++;
1489                         sblock_to_check->data_corrected = 1;
1490                         spin_unlock(&sctx->stat_lock);
1491                         btrfs_err_rl_in_rcu(fs_info,
1492                                 "fixed up error at logical %llu on dev %s",
1493                                 logical, rcu_str_deref(dev->name));
1494                 }
1495         } else {
1496 did_not_correct_error:
1497                 spin_lock(&sctx->stat_lock);
1498                 sctx->stat.uncorrectable_errors++;
1499                 spin_unlock(&sctx->stat_lock);
1500                 btrfs_err_rl_in_rcu(fs_info,
1501                         "unable to fixup (regular) error at logical %llu on dev %s",
1502                         logical, rcu_str_deref(dev->name));
1503         }
1504
1505 out:
1506         if (sblocks_for_recheck) {
1507                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1508                      mirror_index++) {
1509                         struct scrub_block *sblock = sblocks_for_recheck +
1510                                                      mirror_index;
1511                         struct scrub_recover *recover;
1512                         int page_index;
1513
1514                         for (page_index = 0; page_index < sblock->page_count;
1515                              page_index++) {
1516                                 sblock->pagev[page_index]->sblock = NULL;
1517                                 recover = sblock->pagev[page_index]->recover;
1518                                 if (recover) {
1519                                         scrub_put_recover(fs_info, recover);
1520                                         sblock->pagev[page_index]->recover =
1521                                                                         NULL;
1522                                 }
1523                                 scrub_page_put(sblock->pagev[page_index]);
1524                         }
1525                 }
1526                 kfree(sblocks_for_recheck);
1527         }
1528
1529         ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
1530         if (ret < 0)
1531                 return ret;
1532         return 0;
1533 }
1534
1535 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1536 {
1537         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1538                 return 2;
1539         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1540                 return 3;
1541         else
1542                 return (int)bbio->num_stripes;
1543 }
1544
1545 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1546                                                  u64 *raid_map,
1547                                                  u64 mapped_length,
1548                                                  int nstripes, int mirror,
1549                                                  int *stripe_index,
1550                                                  u64 *stripe_offset)
1551 {
1552         int i;
1553
1554         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1555                 /* RAID5/6 */
1556                 for (i = 0; i < nstripes; i++) {
1557                         if (raid_map[i] == RAID6_Q_STRIPE ||
1558                             raid_map[i] == RAID5_P_STRIPE)
1559                                 continue;
1560
1561                         if (logical >= raid_map[i] &&
1562                             logical < raid_map[i] + mapped_length)
1563                                 break;
1564                 }
1565
1566                 *stripe_index = i;
1567                 *stripe_offset = logical - raid_map[i];
1568         } else {
1569                 /* The other RAID type */
1570                 *stripe_index = mirror;
1571                 *stripe_offset = 0;
1572         }
1573 }
1574
1575 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1576                                      struct scrub_block *sblocks_for_recheck)
1577 {
1578         struct scrub_ctx *sctx = original_sblock->sctx;
1579         struct btrfs_fs_info *fs_info = sctx->fs_info;
1580         u64 length = original_sblock->page_count * PAGE_SIZE;
1581         u64 logical = original_sblock->pagev[0]->logical;
1582         u64 generation = original_sblock->pagev[0]->generation;
1583         u64 flags = original_sblock->pagev[0]->flags;
1584         u64 have_csum = original_sblock->pagev[0]->have_csum;
1585         struct scrub_recover *recover;
1586         struct btrfs_bio *bbio;
1587         u64 sublen;
1588         u64 mapped_length;
1589         u64 stripe_offset;
1590         int stripe_index;
1591         int page_index = 0;
1592         int mirror_index;
1593         int nmirrors;
1594         int ret;
1595
1596         /*
1597          * note: the two members refs and outstanding_pages
1598          * are not used (and not set) in the blocks that are used for
1599          * the recheck procedure
1600          */
1601
1602         while (length > 0) {
1603                 sublen = min_t(u64, length, PAGE_SIZE);
1604                 mapped_length = sublen;
1605                 bbio = NULL;
1606
1607                 /*
1608                  * with a length of PAGE_SIZE, each returned stripe
1609                  * represents one mirror
1610                  */
1611                 btrfs_bio_counter_inc_blocked(fs_info);
1612                 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1613                                 logical, &mapped_length, &bbio);
1614                 if (ret || !bbio || mapped_length < sublen) {
1615                         btrfs_put_bbio(bbio);
1616                         btrfs_bio_counter_dec(fs_info);
1617                         return -EIO;
1618                 }
1619
1620                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1621                 if (!recover) {
1622                         btrfs_put_bbio(bbio);
1623                         btrfs_bio_counter_dec(fs_info);
1624                         return -ENOMEM;
1625                 }
1626
1627                 refcount_set(&recover->refs, 1);
1628                 recover->bbio = bbio;
1629                 recover->map_length = mapped_length;
1630
1631                 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1632
1633                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1634
1635                 for (mirror_index = 0; mirror_index < nmirrors;
1636                      mirror_index++) {
1637                         struct scrub_block *sblock;
1638                         struct scrub_page *page;
1639
1640                         sblock = sblocks_for_recheck + mirror_index;
1641                         sblock->sctx = sctx;
1642
1643                         page = kzalloc(sizeof(*page), GFP_NOFS);
1644                         if (!page) {
1645 leave_nomem:
1646                                 spin_lock(&sctx->stat_lock);
1647                                 sctx->stat.malloc_errors++;
1648                                 spin_unlock(&sctx->stat_lock);
1649                                 scrub_put_recover(fs_info, recover);
1650                                 return -ENOMEM;
1651                         }
1652                         scrub_page_get(page);
1653                         sblock->pagev[page_index] = page;
1654                         page->sblock = sblock;
1655                         page->flags = flags;
1656                         page->generation = generation;
1657                         page->logical = logical;
1658                         page->have_csum = have_csum;
1659                         if (have_csum)
1660                                 memcpy(page->csum,
1661                                        original_sblock->pagev[0]->csum,
1662                                        sctx->csum_size);
1663
1664                         scrub_stripe_index_and_offset(logical,
1665                                                       bbio->map_type,
1666                                                       bbio->raid_map,
1667                                                       mapped_length,
1668                                                       bbio->num_stripes -
1669                                                       bbio->num_tgtdevs,
1670                                                       mirror_index,
1671                                                       &stripe_index,
1672                                                       &stripe_offset);
1673                         page->physical = bbio->stripes[stripe_index].physical +
1674                                          stripe_offset;
1675                         page->dev = bbio->stripes[stripe_index].dev;
1676
1677                         BUG_ON(page_index >= original_sblock->page_count);
1678                         page->physical_for_dev_replace =
1679                                 original_sblock->pagev[page_index]->
1680                                 physical_for_dev_replace;
1681                         /* for missing devices, dev->bdev is NULL */
1682                         page->mirror_num = mirror_index + 1;
1683                         sblock->page_count++;
1684                         page->page = alloc_page(GFP_NOFS);
1685                         if (!page->page)
1686                                 goto leave_nomem;
1687
1688                         scrub_get_recover(recover);
1689                         page->recover = recover;
1690                 }
1691                 scrub_put_recover(fs_info, recover);
1692                 length -= sublen;
1693                 logical += sublen;
1694                 page_index++;
1695         }
1696
1697         return 0;
1698 }
1699
1700 static void scrub_bio_wait_endio(struct bio *bio)
1701 {
1702         complete(bio->bi_private);
1703 }
1704
1705 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1706                                         struct bio *bio,
1707                                         struct scrub_page *page)
1708 {
1709         DECLARE_COMPLETION_ONSTACK(done);
1710         int ret;
1711         int mirror_num;
1712
1713         bio->bi_iter.bi_sector = page->logical >> 9;
1714         bio->bi_private = &done;
1715         bio->bi_end_io = scrub_bio_wait_endio;
1716
1717         mirror_num = page->sblock->pagev[0]->mirror_num;
1718         ret = raid56_parity_recover(fs_info, bio, page->recover->bbio,
1719                                     page->recover->map_length,
1720                                     mirror_num, 0);
1721         if (ret)
1722                 return ret;
1723
1724         wait_for_completion_io(&done);
1725         return blk_status_to_errno(bio->bi_status);
1726 }
1727
1728 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
1729                                           struct scrub_block *sblock)
1730 {
1731         struct scrub_page *first_page = sblock->pagev[0];
1732         struct bio *bio;
1733         int page_num;
1734
1735         /* All pages in sblock belong to the same stripe on the same device. */
1736         ASSERT(first_page->dev);
1737         if (!first_page->dev->bdev)
1738                 goto out;
1739
1740         bio = btrfs_io_bio_alloc(BIO_MAX_PAGES);
1741         bio_set_dev(bio, first_page->dev->bdev);
1742
1743         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1744                 struct scrub_page *page = sblock->pagev[page_num];
1745
1746                 WARN_ON(!page->page);
1747                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1748         }
1749
1750         if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
1751                 bio_put(bio);
1752                 goto out;
1753         }
1754
1755         bio_put(bio);
1756
1757         scrub_recheck_block_checksum(sblock);
1758
1759         return;
1760 out:
1761         for (page_num = 0; page_num < sblock->page_count; page_num++)
1762                 sblock->pagev[page_num]->io_error = 1;
1763
1764         sblock->no_io_error_seen = 0;
1765 }
1766
1767 /*
1768  * this function will check the on disk data for checksum errors, header
1769  * errors and read I/O errors. If any I/O errors happen, the exact pages
1770  * which are errored are marked as being bad. The goal is to enable scrub
1771  * to take those pages that are not errored from all the mirrors so that
1772  * the pages that are errored in the just handled mirror can be repaired.
1773  */
1774 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1775                                 struct scrub_block *sblock,
1776                                 int retry_failed_mirror)
1777 {
1778         int page_num;
1779
1780         sblock->no_io_error_seen = 1;
1781
1782         /* short cut for raid56 */
1783         if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
1784                 return scrub_recheck_block_on_raid56(fs_info, sblock);
1785
1786         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1787                 struct bio *bio;
1788                 struct scrub_page *page = sblock->pagev[page_num];
1789
1790                 if (page->dev->bdev == NULL) {
1791                         page->io_error = 1;
1792                         sblock->no_io_error_seen = 0;
1793                         continue;
1794                 }
1795
1796                 WARN_ON(!page->page);
1797                 bio = btrfs_io_bio_alloc(1);
1798                 bio_set_dev(bio, page->dev->bdev);
1799
1800                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1801                 bio->bi_iter.bi_sector = page->physical >> 9;
1802                 bio->bi_opf = REQ_OP_READ;
1803
1804                 if (btrfsic_submit_bio_wait(bio)) {
1805                         page->io_error = 1;
1806                         sblock->no_io_error_seen = 0;
1807                 }
1808
1809                 bio_put(bio);
1810         }
1811
1812         if (sblock->no_io_error_seen)
1813                 scrub_recheck_block_checksum(sblock);
1814 }
1815
1816 static inline int scrub_check_fsid(u8 fsid[],
1817                                    struct scrub_page *spage)
1818 {
1819         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1820         int ret;
1821
1822         ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
1823         return !ret;
1824 }
1825
1826 static void scrub_recheck_block_checksum(struct scrub_block *sblock)
1827 {
1828         sblock->header_error = 0;
1829         sblock->checksum_error = 0;
1830         sblock->generation_error = 0;
1831
1832         if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
1833                 scrub_checksum_data(sblock);
1834         else
1835                 scrub_checksum_tree_block(sblock);
1836 }
1837
1838 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1839                                              struct scrub_block *sblock_good)
1840 {
1841         int page_num;
1842         int ret = 0;
1843
1844         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1845                 int ret_sub;
1846
1847                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1848                                                            sblock_good,
1849                                                            page_num, 1);
1850                 if (ret_sub)
1851                         ret = ret_sub;
1852         }
1853
1854         return ret;
1855 }
1856
1857 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1858                                             struct scrub_block *sblock_good,
1859                                             int page_num, int force_write)
1860 {
1861         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1862         struct scrub_page *page_good = sblock_good->pagev[page_num];
1863         struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
1864
1865         BUG_ON(page_bad->page == NULL);
1866         BUG_ON(page_good->page == NULL);
1867         if (force_write || sblock_bad->header_error ||
1868             sblock_bad->checksum_error || page_bad->io_error) {
1869                 struct bio *bio;
1870                 int ret;
1871
1872                 if (!page_bad->dev->bdev) {
1873                         btrfs_warn_rl(fs_info,
1874                                 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1875                         return -EIO;
1876                 }
1877
1878                 bio = btrfs_io_bio_alloc(1);
1879                 bio_set_dev(bio, page_bad->dev->bdev);
1880                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1881                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1882
1883                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1884                 if (PAGE_SIZE != ret) {
1885                         bio_put(bio);
1886                         return -EIO;
1887                 }
1888
1889                 if (btrfsic_submit_bio_wait(bio)) {
1890                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1891                                 BTRFS_DEV_STAT_WRITE_ERRS);
1892                         btrfs_dev_replace_stats_inc(
1893                                 &fs_info->dev_replace.num_write_errors);
1894                         bio_put(bio);
1895                         return -EIO;
1896                 }
1897                 bio_put(bio);
1898         }
1899
1900         return 0;
1901 }
1902
1903 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1904 {
1905         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
1906         int page_num;
1907
1908         /*
1909          * This block is used for the check of the parity on the source device,
1910          * so the data needn't be written into the destination device.
1911          */
1912         if (sblock->sparity)
1913                 return;
1914
1915         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1916                 int ret;
1917
1918                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1919                 if (ret)
1920                         btrfs_dev_replace_stats_inc(
1921                                 &fs_info->dev_replace.num_write_errors);
1922         }
1923 }
1924
1925 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1926                                            int page_num)
1927 {
1928         struct scrub_page *spage = sblock->pagev[page_num];
1929
1930         BUG_ON(spage->page == NULL);
1931         if (spage->io_error) {
1932                 void *mapped_buffer = kmap_atomic(spage->page);
1933
1934                 clear_page(mapped_buffer);
1935                 flush_dcache_page(spage->page);
1936                 kunmap_atomic(mapped_buffer);
1937         }
1938         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1939 }
1940
1941 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1942                                     struct scrub_page *spage)
1943 {
1944         struct scrub_bio *sbio;
1945         int ret;
1946
1947         mutex_lock(&sctx->wr_lock);
1948 again:
1949         if (!sctx->wr_curr_bio) {
1950                 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
1951                                               GFP_KERNEL);
1952                 if (!sctx->wr_curr_bio) {
1953                         mutex_unlock(&sctx->wr_lock);
1954                         return -ENOMEM;
1955                 }
1956                 sctx->wr_curr_bio->sctx = sctx;
1957                 sctx->wr_curr_bio->page_count = 0;
1958         }
1959         sbio = sctx->wr_curr_bio;
1960         if (sbio->page_count == 0) {
1961                 struct bio *bio;
1962
1963                 sbio->physical = spage->physical_for_dev_replace;
1964                 sbio->logical = spage->logical;
1965                 sbio->dev = sctx->wr_tgtdev;
1966                 bio = sbio->bio;
1967                 if (!bio) {
1968                         bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio);
1969                         sbio->bio = bio;
1970                 }
1971
1972                 bio->bi_private = sbio;
1973                 bio->bi_end_io = scrub_wr_bio_end_io;
1974                 bio_set_dev(bio, sbio->dev->bdev);
1975                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1976                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1977                 sbio->status = 0;
1978         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1979                    spage->physical_for_dev_replace ||
1980                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1981                    spage->logical) {
1982                 scrub_wr_submit(sctx);
1983                 goto again;
1984         }
1985
1986         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1987         if (ret != PAGE_SIZE) {
1988                 if (sbio->page_count < 1) {
1989                         bio_put(sbio->bio);
1990                         sbio->bio = NULL;
1991                         mutex_unlock(&sctx->wr_lock);
1992                         return -EIO;
1993                 }
1994                 scrub_wr_submit(sctx);
1995                 goto again;
1996         }
1997
1998         sbio->pagev[sbio->page_count] = spage;
1999         scrub_page_get(spage);
2000         sbio->page_count++;
2001         if (sbio->page_count == sctx->pages_per_wr_bio)
2002                 scrub_wr_submit(sctx);
2003         mutex_unlock(&sctx->wr_lock);
2004
2005         return 0;
2006 }
2007
2008 static void scrub_wr_submit(struct scrub_ctx *sctx)
2009 {
2010         struct scrub_bio *sbio;
2011
2012         if (!sctx->wr_curr_bio)
2013                 return;
2014
2015         sbio = sctx->wr_curr_bio;
2016         sctx->wr_curr_bio = NULL;
2017         WARN_ON(!sbio->bio->bi_disk);
2018         scrub_pending_bio_inc(sctx);
2019         /* process all writes in a single worker thread. Then the block layer
2020          * orders the requests before sending them to the driver which
2021          * doubled the write performance on spinning disks when measured
2022          * with Linux 3.5 */
2023         btrfsic_submit_bio(sbio->bio);
2024 }
2025
2026 static void scrub_wr_bio_end_io(struct bio *bio)
2027 {
2028         struct scrub_bio *sbio = bio->bi_private;
2029         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2030
2031         sbio->status = bio->bi_status;
2032         sbio->bio = bio;
2033
2034         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
2035                          scrub_wr_bio_end_io_worker, NULL, NULL);
2036         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
2037 }
2038
2039 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
2040 {
2041         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2042         struct scrub_ctx *sctx = sbio->sctx;
2043         int i;
2044
2045         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
2046         if (sbio->status) {
2047                 struct btrfs_dev_replace *dev_replace =
2048                         &sbio->sctx->fs_info->dev_replace;
2049
2050                 for (i = 0; i < sbio->page_count; i++) {
2051                         struct scrub_page *spage = sbio->pagev[i];
2052
2053                         spage->io_error = 1;
2054                         btrfs_dev_replace_stats_inc(&dev_replace->
2055                                                     num_write_errors);
2056                 }
2057         }
2058
2059         for (i = 0; i < sbio->page_count; i++)
2060                 scrub_page_put(sbio->pagev[i]);
2061
2062         bio_put(sbio->bio);
2063         kfree(sbio);
2064         scrub_pending_bio_dec(sctx);
2065 }
2066
2067 static int scrub_checksum(struct scrub_block *sblock)
2068 {
2069         u64 flags;
2070         int ret;
2071
2072         /*
2073          * No need to initialize these stats currently,
2074          * because this function only use return value
2075          * instead of these stats value.
2076          *
2077          * Todo:
2078          * always use stats
2079          */
2080         sblock->header_error = 0;
2081         sblock->generation_error = 0;
2082         sblock->checksum_error = 0;
2083
2084         WARN_ON(sblock->page_count < 1);
2085         flags = sblock->pagev[0]->flags;
2086         ret = 0;
2087         if (flags & BTRFS_EXTENT_FLAG_DATA)
2088                 ret = scrub_checksum_data(sblock);
2089         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
2090                 ret = scrub_checksum_tree_block(sblock);
2091         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
2092                 (void)scrub_checksum_super(sblock);
2093         else
2094                 WARN_ON(1);
2095         if (ret)
2096                 scrub_handle_errored_block(sblock);
2097
2098         return ret;
2099 }
2100
2101 static int scrub_checksum_data(struct scrub_block *sblock)
2102 {
2103         struct scrub_ctx *sctx = sblock->sctx;
2104         u8 csum[BTRFS_CSUM_SIZE];
2105         u8 *on_disk_csum;
2106         struct page *page;
2107         void *buffer;
2108         u32 crc = ~(u32)0;
2109         u64 len;
2110         int index;
2111
2112         BUG_ON(sblock->page_count < 1);
2113         if (!sblock->pagev[0]->have_csum)
2114                 return 0;
2115
2116         on_disk_csum = sblock->pagev[0]->csum;
2117         page = sblock->pagev[0]->page;
2118         buffer = kmap_atomic(page);
2119
2120         len = sctx->fs_info->sectorsize;
2121         index = 0;
2122         for (;;) {
2123                 u64 l = min_t(u64, len, PAGE_SIZE);
2124
2125                 crc = btrfs_csum_data(buffer, crc, l);
2126                 kunmap_atomic(buffer);
2127                 len -= l;
2128                 if (len == 0)
2129                         break;
2130                 index++;
2131                 BUG_ON(index >= sblock->page_count);
2132                 BUG_ON(!sblock->pagev[index]->page);
2133                 page = sblock->pagev[index]->page;
2134                 buffer = kmap_atomic(page);
2135         }
2136
2137         btrfs_csum_final(crc, csum);
2138         if (memcmp(csum, on_disk_csum, sctx->csum_size))
2139                 sblock->checksum_error = 1;
2140
2141         return sblock->checksum_error;
2142 }
2143
2144 static int scrub_checksum_tree_block(struct scrub_block *sblock)
2145 {
2146         struct scrub_ctx *sctx = sblock->sctx;
2147         struct btrfs_header *h;
2148         struct btrfs_fs_info *fs_info = sctx->fs_info;
2149         u8 calculated_csum[BTRFS_CSUM_SIZE];
2150         u8 on_disk_csum[BTRFS_CSUM_SIZE];
2151         struct page *page;
2152         void *mapped_buffer;
2153         u64 mapped_size;
2154         void *p;
2155         u32 crc = ~(u32)0;
2156         u64 len;
2157         int index;
2158
2159         BUG_ON(sblock->page_count < 1);
2160         page = sblock->pagev[0]->page;
2161         mapped_buffer = kmap_atomic(page);
2162         h = (struct btrfs_header *)mapped_buffer;
2163         memcpy(on_disk_csum, h->csum, sctx->csum_size);
2164
2165         /*
2166          * we don't use the getter functions here, as we
2167          * a) don't have an extent buffer and
2168          * b) the page is already kmapped
2169          */
2170         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
2171                 sblock->header_error = 1;
2172
2173         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {
2174                 sblock->header_error = 1;
2175                 sblock->generation_error = 1;
2176         }
2177
2178         if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
2179                 sblock->header_error = 1;
2180
2181         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
2182                    BTRFS_UUID_SIZE))
2183                 sblock->header_error = 1;
2184
2185         len = sctx->fs_info->nodesize - BTRFS_CSUM_SIZE;
2186         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2187         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2188         index = 0;
2189         for (;;) {
2190                 u64 l = min_t(u64, len, mapped_size);
2191
2192                 crc = btrfs_csum_data(p, crc, l);
2193                 kunmap_atomic(mapped_buffer);
2194                 len -= l;
2195                 if (len == 0)
2196                         break;
2197                 index++;
2198                 BUG_ON(index >= sblock->page_count);
2199                 BUG_ON(!sblock->pagev[index]->page);
2200                 page = sblock->pagev[index]->page;
2201                 mapped_buffer = kmap_atomic(page);
2202                 mapped_size = PAGE_SIZE;
2203                 p = mapped_buffer;
2204         }
2205
2206         btrfs_csum_final(crc, calculated_csum);
2207         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2208                 sblock->checksum_error = 1;
2209
2210         return sblock->header_error || sblock->checksum_error;
2211 }
2212
2213 static int scrub_checksum_super(struct scrub_block *sblock)
2214 {
2215         struct btrfs_super_block *s;
2216         struct scrub_ctx *sctx = sblock->sctx;
2217         u8 calculated_csum[BTRFS_CSUM_SIZE];
2218         u8 on_disk_csum[BTRFS_CSUM_SIZE];
2219         struct page *page;
2220         void *mapped_buffer;
2221         u64 mapped_size;
2222         void *p;
2223         u32 crc = ~(u32)0;
2224         int fail_gen = 0;
2225         int fail_cor = 0;
2226         u64 len;
2227         int index;
2228
2229         BUG_ON(sblock->page_count < 1);
2230         page = sblock->pagev[0]->page;
2231         mapped_buffer = kmap_atomic(page);
2232         s = (struct btrfs_super_block *)mapped_buffer;
2233         memcpy(on_disk_csum, s->csum, sctx->csum_size);
2234
2235         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2236                 ++fail_cor;
2237
2238         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2239                 ++fail_gen;
2240
2241         if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2242                 ++fail_cor;
2243
2244         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2245         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2246         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2247         index = 0;
2248         for (;;) {
2249                 u64 l = min_t(u64, len, mapped_size);
2250
2251                 crc = btrfs_csum_data(p, crc, l);
2252                 kunmap_atomic(mapped_buffer);
2253                 len -= l;
2254                 if (len == 0)
2255                         break;
2256                 index++;
2257                 BUG_ON(index >= sblock->page_count);
2258                 BUG_ON(!sblock->pagev[index]->page);
2259                 page = sblock->pagev[index]->page;
2260                 mapped_buffer = kmap_atomic(page);
2261                 mapped_size = PAGE_SIZE;
2262                 p = mapped_buffer;
2263         }
2264
2265         btrfs_csum_final(crc, calculated_csum);
2266         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2267                 ++fail_cor;
2268
2269         if (fail_cor + fail_gen) {
2270                 /*
2271                  * if we find an error in a super block, we just report it.
2272                  * They will get written with the next transaction commit
2273                  * anyway
2274                  */
2275                 spin_lock(&sctx->stat_lock);
2276                 ++sctx->stat.super_errors;
2277                 spin_unlock(&sctx->stat_lock);
2278                 if (fail_cor)
2279                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2280                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2281                 else
2282                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2283                                 BTRFS_DEV_STAT_GENERATION_ERRS);
2284         }
2285
2286         return fail_cor + fail_gen;
2287 }
2288
2289 static void scrub_block_get(struct scrub_block *sblock)
2290 {
2291         refcount_inc(&sblock->refs);
2292 }
2293
2294 static void scrub_block_put(struct scrub_block *sblock)
2295 {
2296         if (refcount_dec_and_test(&sblock->refs)) {
2297                 int i;
2298
2299                 if (sblock->sparity)
2300                         scrub_parity_put(sblock->sparity);
2301
2302                 for (i = 0; i < sblock->page_count; i++)
2303                         scrub_page_put(sblock->pagev[i]);
2304                 kfree(sblock);
2305         }
2306 }
2307
2308 static void scrub_page_get(struct scrub_page *spage)
2309 {
2310         atomic_inc(&spage->refs);
2311 }
2312
2313 static void scrub_page_put(struct scrub_page *spage)
2314 {
2315         if (atomic_dec_and_test(&spage->refs)) {
2316                 if (spage->page)
2317                         __free_page(spage->page);
2318                 kfree(spage);
2319         }
2320 }
2321
2322 static void scrub_submit(struct scrub_ctx *sctx)
2323 {
2324         struct scrub_bio *sbio;
2325
2326         if (sctx->curr == -1)
2327                 return;
2328
2329         sbio = sctx->bios[sctx->curr];
2330         sctx->curr = -1;
2331         scrub_pending_bio_inc(sctx);
2332         btrfsic_submit_bio(sbio->bio);
2333 }
2334
2335 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2336                                     struct scrub_page *spage)
2337 {
2338         struct scrub_block *sblock = spage->sblock;
2339         struct scrub_bio *sbio;
2340         int ret;
2341
2342 again:
2343         /*
2344          * grab a fresh bio or wait for one to become available
2345          */
2346         while (sctx->curr == -1) {
2347                 spin_lock(&sctx->list_lock);
2348                 sctx->curr = sctx->first_free;
2349                 if (sctx->curr != -1) {
2350                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
2351                         sctx->bios[sctx->curr]->next_free = -1;
2352                         sctx->bios[sctx->curr]->page_count = 0;
2353                         spin_unlock(&sctx->list_lock);
2354                 } else {
2355                         spin_unlock(&sctx->list_lock);
2356                         wait_event(sctx->list_wait, sctx->first_free != -1);
2357                 }
2358         }
2359         sbio = sctx->bios[sctx->curr];
2360         if (sbio->page_count == 0) {
2361                 struct bio *bio;
2362
2363                 sbio->physical = spage->physical;
2364                 sbio->logical = spage->logical;
2365                 sbio->dev = spage->dev;
2366                 bio = sbio->bio;
2367                 if (!bio) {
2368                         bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio);
2369                         sbio->bio = bio;
2370                 }
2371
2372                 bio->bi_private = sbio;
2373                 bio->bi_end_io = scrub_bio_end_io;
2374                 bio_set_dev(bio, sbio->dev->bdev);
2375                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2376                 bio_set_op_attrs(bio, REQ_OP_READ, 0);
2377                 sbio->status = 0;
2378         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2379                    spage->physical ||
2380                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2381                    spage->logical ||
2382                    sbio->dev != spage->dev) {
2383                 scrub_submit(sctx);
2384                 goto again;
2385         }
2386
2387         sbio->pagev[sbio->page_count] = spage;
2388         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2389         if (ret != PAGE_SIZE) {
2390                 if (sbio->page_count < 1) {
2391                         bio_put(sbio->bio);
2392                         sbio->bio = NULL;
2393                         return -EIO;
2394                 }
2395                 scrub_submit(sctx);
2396                 goto again;
2397         }
2398
2399         scrub_block_get(sblock); /* one for the page added to the bio */
2400         atomic_inc(&sblock->outstanding_pages);
2401         sbio->page_count++;
2402         if (sbio->page_count == sctx->pages_per_rd_bio)
2403                 scrub_submit(sctx);
2404
2405         return 0;
2406 }
2407
2408 static void scrub_missing_raid56_end_io(struct bio *bio)
2409 {
2410         struct scrub_block *sblock = bio->bi_private;
2411         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
2412
2413         if (bio->bi_status)
2414                 sblock->no_io_error_seen = 0;
2415
2416         bio_put(bio);
2417
2418         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2419 }
2420
2421 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2422 {
2423         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2424         struct scrub_ctx *sctx = sblock->sctx;
2425         struct btrfs_fs_info *fs_info = sctx->fs_info;
2426         u64 logical;
2427         struct btrfs_device *dev;
2428
2429         logical = sblock->pagev[0]->logical;
2430         dev = sblock->pagev[0]->dev;
2431
2432         if (sblock->no_io_error_seen)
2433                 scrub_recheck_block_checksum(sblock);
2434
2435         if (!sblock->no_io_error_seen) {
2436                 spin_lock(&sctx->stat_lock);
2437                 sctx->stat.read_errors++;
2438                 spin_unlock(&sctx->stat_lock);
2439                 btrfs_err_rl_in_rcu(fs_info,
2440                         "IO error rebuilding logical %llu for dev %s",
2441                         logical, rcu_str_deref(dev->name));
2442         } else if (sblock->header_error || sblock->checksum_error) {
2443                 spin_lock(&sctx->stat_lock);
2444                 sctx->stat.uncorrectable_errors++;
2445                 spin_unlock(&sctx->stat_lock);
2446                 btrfs_err_rl_in_rcu(fs_info,
2447                         "failed to rebuild valid logical %llu for dev %s",
2448                         logical, rcu_str_deref(dev->name));
2449         } else {
2450                 scrub_write_block_to_dev_replace(sblock);
2451         }
2452
2453         scrub_block_put(sblock);
2454
2455         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2456                 mutex_lock(&sctx->wr_lock);
2457                 scrub_wr_submit(sctx);
2458                 mutex_unlock(&sctx->wr_lock);
2459         }
2460
2461         scrub_pending_bio_dec(sctx);
2462 }
2463
2464 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2465 {
2466         struct scrub_ctx *sctx = sblock->sctx;
2467         struct btrfs_fs_info *fs_info = sctx->fs_info;
2468         u64 length = sblock->page_count * PAGE_SIZE;
2469         u64 logical = sblock->pagev[0]->logical;
2470         struct btrfs_bio *bbio = NULL;
2471         struct bio *bio;
2472         struct btrfs_raid_bio *rbio;
2473         int ret;
2474         int i;
2475
2476         btrfs_bio_counter_inc_blocked(fs_info);
2477         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
2478                         &length, &bbio);
2479         if (ret || !bbio || !bbio->raid_map)
2480                 goto bbio_out;
2481
2482         if (WARN_ON(!sctx->is_dev_replace ||
2483                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2484                 /*
2485                  * We shouldn't be scrubbing a missing device. Even for dev
2486                  * replace, we should only get here for RAID 5/6. We either
2487                  * managed to mount something with no mirrors remaining or
2488                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2489                  */
2490                 goto bbio_out;
2491         }
2492
2493         bio = btrfs_io_bio_alloc(0);
2494         bio->bi_iter.bi_sector = logical >> 9;
2495         bio->bi_private = sblock;
2496         bio->bi_end_io = scrub_missing_raid56_end_io;
2497
2498         rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length);
2499         if (!rbio)
2500                 goto rbio_out;
2501
2502         for (i = 0; i < sblock->page_count; i++) {
2503                 struct scrub_page *spage = sblock->pagev[i];
2504
2505                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2506         }
2507
2508         btrfs_init_work(&sblock->work, btrfs_scrub_helper,
2509                         scrub_missing_raid56_worker, NULL, NULL);
2510         scrub_block_get(sblock);
2511         scrub_pending_bio_inc(sctx);
2512         raid56_submit_missing_rbio(rbio);
2513         return;
2514
2515 rbio_out:
2516         bio_put(bio);
2517 bbio_out:
2518         btrfs_bio_counter_dec(fs_info);
2519         btrfs_put_bbio(bbio);
2520         spin_lock(&sctx->stat_lock);
2521         sctx->stat.malloc_errors++;
2522         spin_unlock(&sctx->stat_lock);
2523 }
2524
2525 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2526                        u64 physical, struct btrfs_device *dev, u64 flags,
2527                        u64 gen, int mirror_num, u8 *csum, int force,
2528                        u64 physical_for_dev_replace)
2529 {
2530         struct scrub_block *sblock;
2531         int index;
2532
2533         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2534         if (!sblock) {
2535                 spin_lock(&sctx->stat_lock);
2536                 sctx->stat.malloc_errors++;
2537                 spin_unlock(&sctx->stat_lock);
2538                 return -ENOMEM;
2539         }
2540
2541         /* one ref inside this function, plus one for each page added to
2542          * a bio later on */
2543         refcount_set(&sblock->refs, 1);
2544         sblock->sctx = sctx;
2545         sblock->no_io_error_seen = 1;
2546
2547         for (index = 0; len > 0; index++) {
2548                 struct scrub_page *spage;
2549                 u64 l = min_t(u64, len, PAGE_SIZE);
2550
2551                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2552                 if (!spage) {
2553 leave_nomem:
2554                         spin_lock(&sctx->stat_lock);
2555                         sctx->stat.malloc_errors++;
2556                         spin_unlock(&sctx->stat_lock);
2557                         scrub_block_put(sblock);
2558                         return -ENOMEM;
2559                 }
2560                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2561                 scrub_page_get(spage);
2562                 sblock->pagev[index] = spage;
2563                 spage->sblock = sblock;
2564                 spage->dev = dev;
2565                 spage->flags = flags;
2566                 spage->generation = gen;
2567                 spage->logical = logical;
2568                 spage->physical = physical;
2569                 spage->physical_for_dev_replace = physical_for_dev_replace;
2570                 spage->mirror_num = mirror_num;
2571                 if (csum) {
2572                         spage->have_csum = 1;
2573                         memcpy(spage->csum, csum, sctx->csum_size);
2574                 } else {
2575                         spage->have_csum = 0;
2576                 }
2577                 sblock->page_count++;
2578                 spage->page = alloc_page(GFP_KERNEL);
2579                 if (!spage->page)
2580                         goto leave_nomem;
2581                 len -= l;
2582                 logical += l;
2583                 physical += l;
2584                 physical_for_dev_replace += l;
2585         }
2586
2587         WARN_ON(sblock->page_count == 0);
2588         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2589                 /*
2590                  * This case should only be hit for RAID 5/6 device replace. See
2591                  * the comment in scrub_missing_raid56_pages() for details.
2592                  */
2593                 scrub_missing_raid56_pages(sblock);
2594         } else {
2595                 for (index = 0; index < sblock->page_count; index++) {
2596                         struct scrub_page *spage = sblock->pagev[index];
2597                         int ret;
2598
2599                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2600                         if (ret) {
2601                                 scrub_block_put(sblock);
2602                                 return ret;
2603                         }
2604                 }
2605
2606                 if (force)
2607                         scrub_submit(sctx);
2608         }
2609
2610         /* last one frees, either here or in bio completion for last page */
2611         scrub_block_put(sblock);
2612         return 0;
2613 }
2614
2615 static void scrub_bio_end_io(struct bio *bio)
2616 {
2617         struct scrub_bio *sbio = bio->bi_private;
2618         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2619
2620         sbio->status = bio->bi_status;
2621         sbio->bio = bio;
2622
2623         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2624 }
2625
2626 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2627 {
2628         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2629         struct scrub_ctx *sctx = sbio->sctx;
2630         int i;
2631
2632         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2633         if (sbio->status) {
2634                 for (i = 0; i < sbio->page_count; i++) {
2635                         struct scrub_page *spage = sbio->pagev[i];
2636
2637                         spage->io_error = 1;
2638                         spage->sblock->no_io_error_seen = 0;
2639                 }
2640         }
2641
2642         /* now complete the scrub_block items that have all pages completed */
2643         for (i = 0; i < sbio->page_count; i++) {
2644                 struct scrub_page *spage = sbio->pagev[i];
2645                 struct scrub_block *sblock = spage->sblock;
2646
2647                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2648                         scrub_block_complete(sblock);
2649                 scrub_block_put(sblock);
2650         }
2651
2652         bio_put(sbio->bio);
2653         sbio->bio = NULL;
2654         spin_lock(&sctx->list_lock);
2655         sbio->next_free = sctx->first_free;
2656         sctx->first_free = sbio->index;
2657         spin_unlock(&sctx->list_lock);
2658
2659         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2660                 mutex_lock(&sctx->wr_lock);
2661                 scrub_wr_submit(sctx);
2662                 mutex_unlock(&sctx->wr_lock);
2663         }
2664
2665         scrub_pending_bio_dec(sctx);
2666 }
2667
2668 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2669                                        unsigned long *bitmap,
2670                                        u64 start, u64 len)
2671 {
2672         u64 offset;
2673         u64 nsectors64;
2674         u32 nsectors;
2675         int sectorsize = sparity->sctx->fs_info->sectorsize;
2676
2677         if (len >= sparity->stripe_len) {
2678                 bitmap_set(bitmap, 0, sparity->nsectors);
2679                 return;
2680         }
2681
2682         start -= sparity->logic_start;
2683         start = div64_u64_rem(start, sparity->stripe_len, &offset);
2684         offset = div_u64(offset, sectorsize);
2685         nsectors64 = div_u64(len, sectorsize);
2686
2687         ASSERT(nsectors64 < UINT_MAX);
2688         nsectors = (u32)nsectors64;
2689
2690         if (offset + nsectors <= sparity->nsectors) {
2691                 bitmap_set(bitmap, offset, nsectors);
2692                 return;
2693         }
2694
2695         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2696         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2697 }
2698
2699 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2700                                                    u64 start, u64 len)
2701 {
2702         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2703 }
2704
2705 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2706                                                   u64 start, u64 len)
2707 {
2708         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2709 }
2710
2711 static void scrub_block_complete(struct scrub_block *sblock)
2712 {
2713         int corrupted = 0;
2714
2715         if (!sblock->no_io_error_seen) {
2716                 corrupted = 1;
2717                 scrub_handle_errored_block(sblock);
2718         } else {
2719                 /*
2720                  * if has checksum error, write via repair mechanism in
2721                  * dev replace case, otherwise write here in dev replace
2722                  * case.
2723                  */
2724                 corrupted = scrub_checksum(sblock);
2725                 if (!corrupted && sblock->sctx->is_dev_replace)
2726                         scrub_write_block_to_dev_replace(sblock);
2727         }
2728
2729         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2730                 u64 start = sblock->pagev[0]->logical;
2731                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2732                           PAGE_SIZE;
2733
2734                 scrub_parity_mark_sectors_error(sblock->sparity,
2735                                                 start, end - start);
2736         }
2737 }
2738
2739 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
2740 {
2741         struct btrfs_ordered_sum *sum = NULL;
2742         unsigned long index;
2743         unsigned long num_sectors;
2744
2745         while (!list_empty(&sctx->csum_list)) {
2746                 sum = list_first_entry(&sctx->csum_list,
2747                                        struct btrfs_ordered_sum, list);
2748                 if (sum->bytenr > logical)
2749                         return 0;
2750                 if (sum->bytenr + sum->len > logical)
2751                         break;
2752
2753                 ++sctx->stat.csum_discards;
2754                 list_del(&sum->list);
2755                 kfree(sum);
2756                 sum = NULL;
2757         }
2758         if (!sum)
2759                 return 0;
2760
2761         index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize);
2762         ASSERT(index < UINT_MAX);
2763
2764         num_sectors = sum->len / sctx->fs_info->sectorsize;
2765         memcpy(csum, sum->sums + index, sctx->csum_size);
2766         if (index == num_sectors - 1) {
2767                 list_del(&sum->list);
2768                 kfree(sum);
2769         }
2770         return 1;
2771 }
2772
2773 /* scrub extent tries to collect up to 64 kB for each bio */
2774 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
2775                         u64 logical, u64 len,
2776                         u64 physical, struct btrfs_device *dev, u64 flags,
2777                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2778 {
2779         int ret;
2780         u8 csum[BTRFS_CSUM_SIZE];
2781         u32 blocksize;
2782
2783         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2784                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2785                         blocksize = map->stripe_len;
2786                 else
2787                         blocksize = sctx->fs_info->sectorsize;
2788                 spin_lock(&sctx->stat_lock);
2789                 sctx->stat.data_extents_scrubbed++;
2790                 sctx->stat.data_bytes_scrubbed += len;
2791                 spin_unlock(&sctx->stat_lock);
2792         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2793                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2794                         blocksize = map->stripe_len;
2795                 else
2796                         blocksize = sctx->fs_info->nodesize;
2797                 spin_lock(&sctx->stat_lock);
2798                 sctx->stat.tree_extents_scrubbed++;
2799                 sctx->stat.tree_bytes_scrubbed += len;
2800                 spin_unlock(&sctx->stat_lock);
2801         } else {
2802                 blocksize = sctx->fs_info->sectorsize;
2803                 WARN_ON(1);
2804         }
2805
2806         while (len) {
2807                 u64 l = min_t(u64, len, blocksize);
2808                 int have_csum = 0;
2809
2810                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2811                         /* push csums to sbio */
2812                         have_csum = scrub_find_csum(sctx, logical, csum);
2813                         if (have_csum == 0)
2814                                 ++sctx->stat.no_csum;
2815                         if (sctx->is_dev_replace && !have_csum) {
2816                                 ret = copy_nocow_pages(sctx, logical, l,
2817                                                        mirror_num,
2818                                                       physical_for_dev_replace);
2819                                 goto behind_scrub_pages;
2820                         }
2821                 }
2822                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2823                                   mirror_num, have_csum ? csum : NULL, 0,
2824                                   physical_for_dev_replace);
2825 behind_scrub_pages:
2826                 if (ret)
2827                         return ret;
2828                 len -= l;
2829                 logical += l;
2830                 physical += l;
2831                 physical_for_dev_replace += l;
2832         }
2833         return 0;
2834 }
2835
2836 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2837                                   u64 logical, u64 len,
2838                                   u64 physical, struct btrfs_device *dev,
2839                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2840 {
2841         struct scrub_ctx *sctx = sparity->sctx;
2842         struct scrub_block *sblock;
2843         int index;
2844
2845         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2846         if (!sblock) {
2847                 spin_lock(&sctx->stat_lock);
2848                 sctx->stat.malloc_errors++;
2849                 spin_unlock(&sctx->stat_lock);
2850                 return -ENOMEM;
2851         }
2852
2853         /* one ref inside this function, plus one for each page added to
2854          * a bio later on */
2855         refcount_set(&sblock->refs, 1);
2856         sblock->sctx = sctx;
2857         sblock->no_io_error_seen = 1;
2858         sblock->sparity = sparity;
2859         scrub_parity_get(sparity);
2860
2861         for (index = 0; len > 0; index++) {
2862                 struct scrub_page *spage;
2863                 u64 l = min_t(u64, len, PAGE_SIZE);
2864
2865                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2866                 if (!spage) {
2867 leave_nomem:
2868                         spin_lock(&sctx->stat_lock);
2869                         sctx->stat.malloc_errors++;
2870                         spin_unlock(&sctx->stat_lock);
2871                         scrub_block_put(sblock);
2872                         return -ENOMEM;
2873                 }
2874                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2875                 /* For scrub block */
2876                 scrub_page_get(spage);
2877                 sblock->pagev[index] = spage;
2878                 /* For scrub parity */
2879                 scrub_page_get(spage);
2880                 list_add_tail(&spage->list, &sparity->spages);
2881                 spage->sblock = sblock;
2882                 spage->dev = dev;
2883                 spage->flags = flags;
2884                 spage->generation = gen;
2885                 spage->logical = logical;
2886                 spage->physical = physical;
2887                 spage->mirror_num = mirror_num;
2888                 if (csum) {
2889                         spage->have_csum = 1;
2890                         memcpy(spage->csum, csum, sctx->csum_size);
2891                 } else {
2892                         spage->have_csum = 0;
2893                 }
2894                 sblock->page_count++;
2895                 spage->page = alloc_page(GFP_KERNEL);
2896                 if (!spage->page)
2897                         goto leave_nomem;
2898                 len -= l;
2899                 logical += l;
2900                 physical += l;
2901         }
2902
2903         WARN_ON(sblock->page_count == 0);
2904         for (index = 0; index < sblock->page_count; index++) {
2905                 struct scrub_page *spage = sblock->pagev[index];
2906                 int ret;
2907
2908                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2909                 if (ret) {
2910                         scrub_block_put(sblock);
2911                         return ret;
2912                 }
2913         }
2914
2915         /* last one frees, either here or in bio completion for last page */
2916         scrub_block_put(sblock);
2917         return 0;
2918 }
2919
2920 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2921                                    u64 logical, u64 len,
2922                                    u64 physical, struct btrfs_device *dev,
2923                                    u64 flags, u64 gen, int mirror_num)
2924 {
2925         struct scrub_ctx *sctx = sparity->sctx;
2926         int ret;
2927         u8 csum[BTRFS_CSUM_SIZE];
2928         u32 blocksize;
2929
2930         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2931                 scrub_parity_mark_sectors_error(sparity, logical, len);
2932                 return 0;
2933         }
2934
2935         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2936                 blocksize = sparity->stripe_len;
2937         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2938                 blocksize = sparity->stripe_len;
2939         } else {
2940                 blocksize = sctx->fs_info->sectorsize;
2941                 WARN_ON(1);
2942         }
2943
2944         while (len) {
2945                 u64 l = min_t(u64, len, blocksize);
2946                 int have_csum = 0;
2947
2948                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2949                         /* push csums to sbio */
2950                         have_csum = scrub_find_csum(sctx, logical, csum);
2951                         if (have_csum == 0)
2952                                 goto skip;
2953                 }
2954                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2955                                              flags, gen, mirror_num,
2956                                              have_csum ? csum : NULL);
2957                 if (ret)
2958                         return ret;
2959 skip:
2960                 len -= l;
2961                 logical += l;
2962                 physical += l;
2963         }
2964         return 0;
2965 }
2966
2967 /*
2968  * Given a physical address, this will calculate it's
2969  * logical offset. if this is a parity stripe, it will return
2970  * the most left data stripe's logical offset.
2971  *
2972  * return 0 if it is a data stripe, 1 means parity stripe.
2973  */
2974 static int get_raid56_logic_offset(u64 physical, int num,
2975                                    struct map_lookup *map, u64 *offset,
2976                                    u64 *stripe_start)
2977 {
2978         int i;
2979         int j = 0;
2980         u64 stripe_nr;
2981         u64 last_offset;
2982         u32 stripe_index;
2983         u32 rot;
2984
2985         last_offset = (physical - map->stripes[num].physical) *
2986                       nr_data_stripes(map);
2987         if (stripe_start)
2988                 *stripe_start = last_offset;
2989
2990         *offset = last_offset;
2991         for (i = 0; i < nr_data_stripes(map); i++) {
2992                 *offset = last_offset + i * map->stripe_len;
2993
2994                 stripe_nr = div64_u64(*offset, map->stripe_len);
2995                 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2996
2997                 /* Work out the disk rotation on this stripe-set */
2998                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2999                 /* calculate which stripe this data locates */
3000                 rot += i;
3001                 stripe_index = rot % map->num_stripes;
3002                 if (stripe_index == num)
3003                         return 0;
3004                 if (stripe_index < num)
3005                         j++;
3006         }
3007         *offset = last_offset + j * map->stripe_len;
3008         return 1;
3009 }
3010
3011 static void scrub_free_parity(struct scrub_parity *sparity)
3012 {
3013         struct scrub_ctx *sctx = sparity->sctx;
3014         struct scrub_page *curr, *next;
3015         int nbits;
3016
3017         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
3018         if (nbits) {
3019                 spin_lock(&sctx->stat_lock);
3020                 sctx->stat.read_errors += nbits;
3021                 sctx->stat.uncorrectable_errors += nbits;
3022                 spin_unlock(&sctx->stat_lock);
3023         }
3024
3025         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
3026                 list_del_init(&curr->list);
3027                 scrub_page_put(curr);
3028         }
3029
3030         kfree(sparity);
3031 }
3032
3033 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
3034 {
3035         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
3036                                                     work);
3037         struct scrub_ctx *sctx = sparity->sctx;
3038
3039         scrub_free_parity(sparity);
3040         scrub_pending_bio_dec(sctx);
3041 }
3042
3043 static void scrub_parity_bio_endio(struct bio *bio)
3044 {
3045         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
3046         struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
3047
3048         if (bio->bi_status)
3049                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
3050                           sparity->nsectors);
3051
3052         bio_put(bio);
3053
3054         btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
3055                         scrub_parity_bio_endio_worker, NULL, NULL);
3056         btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
3057 }
3058
3059 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
3060 {
3061         struct scrub_ctx *sctx = sparity->sctx;
3062         struct btrfs_fs_info *fs_info = sctx->fs_info;
3063         struct bio *bio;
3064         struct btrfs_raid_bio *rbio;
3065         struct btrfs_bio *bbio = NULL;
3066         u64 length;
3067         int ret;
3068
3069         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
3070                            sparity->nsectors))
3071                 goto out;
3072
3073         length = sparity->logic_end - sparity->logic_start;
3074
3075         btrfs_bio_counter_inc_blocked(fs_info);
3076         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
3077                                &length, &bbio);
3078         if (ret || !bbio || !bbio->raid_map)
3079                 goto bbio_out;
3080
3081         bio = btrfs_io_bio_alloc(0);
3082         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
3083         bio->bi_private = sparity;
3084         bio->bi_end_io = scrub_parity_bio_endio;
3085
3086         rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio,
3087                                               length, sparity->scrub_dev,
3088                                               sparity->dbitmap,
3089                                               sparity->nsectors);
3090         if (!rbio)
3091                 goto rbio_out;
3092
3093         scrub_pending_bio_inc(sctx);
3094         raid56_parity_submit_scrub_rbio(rbio);
3095         return;
3096
3097 rbio_out:
3098         bio_put(bio);
3099 bbio_out:
3100         btrfs_bio_counter_dec(fs_info);
3101         btrfs_put_bbio(bbio);
3102         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
3103                   sparity->nsectors);
3104         spin_lock(&sctx->stat_lock);
3105         sctx->stat.malloc_errors++;
3106         spin_unlock(&sctx->stat_lock);
3107 out:
3108         scrub_free_parity(sparity);
3109 }
3110
3111 static inline int scrub_calc_parity_bitmap_len(int nsectors)
3112 {
3113         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
3114 }
3115
3116 static void scrub_parity_get(struct scrub_parity *sparity)
3117 {
3118         refcount_inc(&sparity->refs);
3119 }
3120
3121 static void scrub_parity_put(struct scrub_parity *sparity)
3122 {
3123         if (!refcount_dec_and_test(&sparity->refs))
3124                 return;
3125
3126         scrub_parity_check_and_repair(sparity);
3127 }
3128
3129 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
3130                                                   struct map_lookup *map,
3131                                                   struct btrfs_device *sdev,
3132                                                   struct btrfs_path *path,
3133                                                   u64 logic_start,
3134                                                   u64 logic_end)
3135 {
3136         struct btrfs_fs_info *fs_info = sctx->fs_info;
3137         struct btrfs_root *root = fs_info->extent_root;
3138         struct btrfs_root *csum_root = fs_info->csum_root;
3139         struct btrfs_extent_item *extent;
3140         struct btrfs_bio *bbio = NULL;
3141         u64 flags;
3142         int ret;
3143         int slot;
3144         struct extent_buffer *l;
3145         struct btrfs_key key;
3146         u64 generation;
3147         u64 extent_logical;
3148         u64 extent_physical;
3149         u64 extent_len;
3150         u64 mapped_length;
3151         struct btrfs_device *extent_dev;
3152         struct scrub_parity *sparity;
3153         int nsectors;
3154         int bitmap_len;
3155         int extent_mirror_num;
3156         int stop_loop = 0;
3157
3158         nsectors = div_u64(map->stripe_len, fs_info->sectorsize);
3159         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
3160         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
3161                           GFP_NOFS);
3162         if (!sparity) {
3163                 spin_lock(&sctx->stat_lock);
3164                 sctx->stat.malloc_errors++;
3165                 spin_unlock(&sctx->stat_lock);
3166                 return -ENOMEM;
3167         }
3168
3169         sparity->stripe_len = map->stripe_len;
3170         sparity->nsectors = nsectors;
3171         sparity->sctx = sctx;
3172         sparity->scrub_dev = sdev;
3173         sparity->logic_start = logic_start;
3174         sparity->logic_end = logic_end;
3175         refcount_set(&sparity->refs, 1);
3176         INIT_LIST_HEAD(&sparity->spages);
3177         sparity->dbitmap = sparity->bitmap;
3178         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
3179
3180         ret = 0;
3181         while (logic_start < logic_end) {
3182                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3183                         key.type = BTRFS_METADATA_ITEM_KEY;
3184                 else
3185                         key.type = BTRFS_EXTENT_ITEM_KEY;
3186                 key.objectid = logic_start;
3187                 key.offset = (u64)-1;
3188
3189                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3190                 if (ret < 0)
3191                         goto out;
3192
3193                 if (ret > 0) {
3194                         ret = btrfs_previous_extent_item(root, path, 0);
3195