fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include <linux/sched/mm.h>
  22 #include "ctree.h"
  23 #include "volumes.h"
  24 #include "disk-io.h"
  25 #include "ordered-data.h"
  26 #include "transaction.h"
  27 #include "backref.h"
  28 #include "extent_io.h"
  29 #include "dev-replace.h"
  30 #include "check-integrity.h"
  31 #include "rcu-string.h"
  32 #include "raid56.h"
  33
  34 /*
  35  * This is only the first step towards a full-features scrub. It reads all
  36  * extent and super block and verifies the checksums. In case a bad checksum
  37  * is found or the extent cannot be read, good data will be written back if
  38  * any can be found.
  39  *
  40  * Future enhancements:
  41  *  - In case an unrepairable extent is encountered, track which files are
  42  *    affected and report them
  43  *  - track and record media errors, throw out bad devices
  44  *  - add a mode to also read unallocated space
  45  */
  46
  47 struct scrub_block;
  48 struct scrub_ctx;
  49
  50 /*
  51  * the following three values only influence the performance.
  52  * The last one configures the number of parallel and outstanding I/O
  53  * operations. The first two values configure an upper limit for the number
  54  * of (dynamically allocated) pages that are added to a bio.
  55  */
  56 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  57 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  58 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  59
  60 /*
  61  * the following value times PAGE_SIZE needs to be large enough to match the
  62  * largest node/leaf/sector size that shall be supported.
  63  * Values larger than BTRFS_STRIPE_LEN are not supported.
  64  */
  65 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  66
  67 struct scrub_recover {
  68         refcount_t              refs;
  69         struct btrfs_bio        *bbio;
  70         u64                     map_length;
  71 };
  72
  73 struct scrub_page {
  74         struct scrub_block      *sblock;
  75         struct page             *page;
  76         struct btrfs_device     *dev;
  77         struct list_head        list;
  78         u64                     flags;  /* extent flags */
  79         u64                     generation;
  80         u64                     logical;
  81         u64                     physical;
  82         u64                     physical_for_dev_replace;
  83         atomic_t                refs;
  84         struct {
  85                 unsigned int    mirror_num:8;
  86                 unsigned int    have_csum:1;
  87                 unsigned int    io_error:1;
  88         };
  89         u8                      csum[BTRFS_CSUM_SIZE];
  90
  91         struct scrub_recover    *recover;
  92 };
  93
  94 struct scrub_bio {
  95         int                     index;
  96         struct scrub_ctx        *sctx;
  97         struct btrfs_device     *dev;
  98         struct bio              *bio;
  99         blk_status_t            status;
 100         u64                     logical;
 101         u64                     physical;
 102 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
 103         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
 104 #else
 105         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
 106 #endif
 107         int                     page_count;
 108         int                     next_free;
 109         struct btrfs_work       work;
 110 };
 111
 112 struct scrub_block {
 113         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 114         int                     page_count;
 115         atomic_t                outstanding_pages;
 116         refcount_t              refs; /* free mem on transition to zero */
 117         struct scrub_ctx        *sctx;
 118         struct scrub_parity     *sparity;
 119         struct {
 120                 unsigned int    header_error:1;
 121                 unsigned int    checksum_error:1;
 122                 unsigned int    no_io_error_seen:1;
 123                 unsigned int    generation_error:1; /* also sets header_error */
 124
 125                 /* The following is for the data used to check parity */
 126                 /* It is for the data with checksum */
 127                 unsigned int    data_corrected:1;
 128         };
 129         struct btrfs_work       work;
 130 };
 131
 132 /* Used for the chunks with parity stripe such RAID5/6 */
 133 struct scrub_parity {
 134         struct scrub_ctx        *sctx;
 135
 136         struct btrfs_device     *scrub_dev;
 137
 138         u64                     logic_start;
 139
 140         u64                     logic_end;
 141
 142         int                     nsectors;
 143
 144         u64                     stripe_len;
 145
 146         refcount_t              refs;
 147
 148         struct list_head        spages;
 149
 150         /* Work of parity check and repair */
 151         struct btrfs_work       work;
 152
 153         /* Mark the parity blocks which have data */
 154         unsigned long           *dbitmap;
 155
 156         /*
 157          * Mark the parity blocks which have data, but errors happen when
 158          * read data or check data
 159          */
 160         unsigned long           *ebitmap;
 161
 162         unsigned long           bitmap[0];
 163 };
 164
 165 struct scrub_ctx {
 166         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 167         struct btrfs_fs_info    *fs_info;
 168         int                     first_free;
 169         int                     curr;
 170         atomic_t                bios_in_flight;
 171         atomic_t                workers_pending;
 172         spinlock_t              list_lock;
 173         wait_queue_head_t       list_wait;
 174         u16                     csum_size;
 175         struct list_head        csum_list;
 176         atomic_t                cancel_req;
 177         int                     readonly;
 178         int                     pages_per_rd_bio;
 179
 180         int                     is_dev_replace;
 181
 182         struct scrub_bio        *wr_curr_bio;
 183         struct mutex            wr_lock;
 184         int                     pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 185         struct btrfs_device     *wr_tgtdev;
 186         bool                    flush_all_writes;
 187
 188         /*
 189          * statistics
 190          */
 191         struct btrfs_scrub_progress stat;
 192         spinlock_t              stat_lock;
 193
 194         /*
 195          * Use a ref counter to avoid use-after-free issues. Scrub workers
 196          * decrement bios_in_flight and workers_pending and then do a wakeup
 197          * on the list_wait wait queue. We must ensure the main scrub task
 198          * doesn't free the scrub context before or while the workers are
 199          * doing the wakeup() call.
 200          */
 201         refcount_t              refs;
 202 };
 203
 204 struct scrub_fixup_nodatasum {
 205         struct scrub_ctx        *sctx;
 206         struct btrfs_device     *dev;
 207         u64                     logical;
 208         struct btrfs_root       *root;
 209         struct btrfs_work       work;
 210         int                     mirror_num;
 211 };
 212
 213 struct scrub_nocow_inode {
 214         u64                     inum;
 215         u64                     offset;
 216         u64                     root;
 217         struct list_head        list;
 218 };
 219
 220 struct scrub_copy_nocow_ctx {
 221         struct scrub_ctx        *sctx;
 222         u64                     logical;
 223         u64                     len;
 224         int                     mirror_num;
 225         u64                     physical_for_dev_replace;
 226         struct list_head        inodes;
 227         struct btrfs_work       work;
 228 };
 229
 230 struct scrub_warning {
 231         struct btrfs_path       *path;
 232         u64                     extent_item_size;
 233         const char              *errstr;
 234         u64                     physical;
 235         u64                     logical;
 236         struct btrfs_device     *dev;
 237 };
 238
 239 struct full_stripe_lock {
 240         struct rb_node node;
 241         u64 logical;
 242         u64 refs;
 243         struct mutex mutex;
 244 };
 245
 246 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 247 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 248 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 249 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 250 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 251 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
 252                                      struct scrub_block *sblocks_for_recheck);
 253 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 254                                 struct scrub_block *sblock,
 255                                 int retry_failed_mirror);
 256 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
 257 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 258                                              struct scrub_block *sblock_good);
 259 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 260                                             struct scrub_block *sblock_good,
 261                                             int page_num, int force_write);
 262 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 263 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 264                                            int page_num);
 265 static int scrub_checksum_data(struct scrub_block *sblock);
 266 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 267 static int scrub_checksum_super(struct scrub_block *sblock);
 268 static void scrub_block_get(struct scrub_block *sblock);
 269 static void scrub_block_put(struct scrub_block *sblock);
 270 static void scrub_page_get(struct scrub_page *spage);
 271 static void scrub_page_put(struct scrub_page *spage);
 272 static void scrub_parity_get(struct scrub_parity *sparity);
 273 static void scrub_parity_put(struct scrub_parity *sparity);
 274 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 275                                     struct scrub_page *spage);
 276 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 277                        u64 physical, struct btrfs_device *dev, u64 flags,
 278                        u64 gen, int mirror_num, u8 *csum, int force,
 279                        u64 physical_for_dev_replace);
 280 static void scrub_bio_end_io(struct bio *bio);
 281 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 282 static void scrub_block_complete(struct scrub_block *sblock);
 283 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 284                                u64 extent_logical, u64 extent_len,
 285                                u64 *extent_physical,
 286                                struct btrfs_device **extent_dev,
 287                                int *extent_mirror_num);
 288 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 289                                     struct scrub_page *spage);
 290 static void scrub_wr_submit(struct scrub_ctx *sctx);
 291 static void scrub_wr_bio_end_io(struct bio *bio);
 292 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 293 static int write_page_nocow(struct scrub_ctx *sctx,
 294                             u64 physical_for_dev_replace, struct page *page);
 295 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 296                                       struct scrub_copy_nocow_ctx *ctx);
 297 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 298                             int mirror_num, u64 physical_for_dev_replace);
 299 static void copy_nocow_pages_worker(struct btrfs_work *work);
 300 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 301 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 302 static void scrub_put_ctx(struct scrub_ctx *sctx);
 303
 304 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
 305 {
 306         return page->recover &&
 307                (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
 308 }
 309
 310 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 311 {
 312         refcount_inc(&sctx->refs);
 313         atomic_inc(&sctx->bios_in_flight);
 314 }
 315
 316 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 317 {
 318         atomic_dec(&sctx->bios_in_flight);
 319         wake_up(&sctx->list_wait);
 320         scrub_put_ctx(sctx);
 321 }
 322
 323 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 324 {
 325         while (atomic_read(&fs_info->scrub_pause_req)) {
 326                 mutex_unlock(&fs_info->scrub_lock);
 327                 wait_event(fs_info->scrub_pause_wait,
 328                    atomic_read(&fs_info->scrub_pause_req) == 0);
 329                 mutex_lock(&fs_info->scrub_lock);
 330         }
 331 }
 332
 333 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
 334 {
 335         atomic_inc(&fs_info->scrubs_paused);
 336         wake_up(&fs_info->scrub_pause_wait);
 337 }
 338
 339 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
 340 {
 341         mutex_lock(&fs_info->scrub_lock);
 342         __scrub_blocked_if_needed(fs_info);
 343         atomic_dec(&fs_info->scrubs_paused);
 344         mutex_unlock(&fs_info->scrub_lock);
 345
 346         wake_up(&fs_info->scrub_pause_wait);
 347 }
 348
 349 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 350 {
 351         scrub_pause_on(fs_info);
 352         scrub_pause_off(fs_info);
 353 }
 354
 355 /*
 356  * Insert new full stripe lock into full stripe locks tree
 357  *
 358  * Return pointer to existing or newly inserted full_stripe_lock structure if
 359  * everything works well.
 360  * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
 361  *
 362  * NOTE: caller must hold full_stripe_locks_root->lock before calling this
 363  * function
 364  */
 365 static struct full_stripe_lock *insert_full_stripe_lock(
 366                 struct btrfs_full_stripe_locks_tree *locks_root,
 367                 u64 fstripe_logical)
 368 {
 369         struct rb_node **p;
 370         struct rb_node *parent = NULL;
 371         struct full_stripe_lock *entry;
 372         struct full_stripe_lock *ret;
 373
 374         lockdep_assert_held(&locks_root->lock);
 375
 376         p = &locks_root->root.rb_node;
 377         while (*p) {
 378                 parent = *p;
 379                 entry = rb_entry(parent, struct full_stripe_lock, node);
 380                 if (fstripe_logical < entry->logical) {
 381                         p = &(*p)->rb_left;
 382                 } else if (fstripe_logical > entry->logical) {
 383                         p = &(*p)->rb_right;
 384                 } else {
 385                         entry->refs++;
 386                         return entry;
 387                 }
 388         }
 389
 390         /* Insert new lock */
 391         ret = kmalloc(sizeof(*ret), GFP_KERNEL);
 392         if (!ret)
 393                 return ERR_PTR(-ENOMEM);
 394         ret->logical = fstripe_logical;
 395         ret->refs = 1;
 396         mutex_init(&ret->mutex);
 397
 398         rb_link_node(&ret->node, parent, p);
 399         rb_insert_color(&ret->node, &locks_root->root);
 400         return ret;
 401 }
 402
 403 /*
 404  * Search for a full stripe lock of a block group
 405  *
 406  * Return pointer to existing full stripe lock if found
 407  * Return NULL if not found
 408  */
 409 static struct full_stripe_lock *search_full_stripe_lock(
 410                 struct btrfs_full_stripe_locks_tree *locks_root,
 411                 u64 fstripe_logical)
 412 {
 413         struct rb_node *node;
 414         struct full_stripe_lock *entry;
 415
 416         lockdep_assert_held(&locks_root->lock);
 417
 418         node = locks_root->root.rb_node;
 419         while (node) {
 420                 entry = rb_entry(node, struct full_stripe_lock, node);
 421                 if (fstripe_logical < entry->logical)
 422                         node = node->rb_left;
 423                 else if (fstripe_logical > entry->logical)
 424                         node = node->rb_right;
 425                 else
 426                         return entry;
 427         }
 428         return NULL;
 429 }
 430
 431 /*
 432  * Helper to get full stripe logical from a normal bytenr.
 433  *
 434  * Caller must ensure @cache is a RAID56 block group.
 435  */
 436 static u64 get_full_stripe_logical(struct btrfs_block_group_cache *cache,
 437                                    u64 bytenr)
 438 {
 439         u64 ret;
 440
 441         /*
 442          * Due to chunk item size limit, full stripe length should not be
 443          * larger than U32_MAX. Just a sanity check here.
 444          */
 445         WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);
 446
 447         /*
 448          * round_down() can only handle power of 2, while RAID56 full
 449          * stripe length can be 64KiB * n, so we need to manually round down.
 450          */
 451         ret = div64_u64(bytenr - cache->key.objectid, cache->full_stripe_len) *
 452                 cache->full_stripe_len + cache->key.objectid;
 453         return ret;
 454 }
 455
 456 /*
 457  * Lock a full stripe to avoid concurrency of recovery and read
 458  *
 459  * It's only used for profiles with parities (RAID5/6), for other profiles it
 460  * does nothing.
 461  *
 462  * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
 463  * So caller must call unlock_full_stripe() at the same context.
 464  *
 465  * Return <0 if encounters error.
 466  */
 467 static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
 468                             bool *locked_ret)
 469 {
 470         struct btrfs_block_group_cache *bg_cache;
 471         struct btrfs_full_stripe_locks_tree *locks_root;
 472         struct full_stripe_lock *existing;
 473         u64 fstripe_start;
 474         int ret = 0;
 475
 476         *locked_ret = false;
 477         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
 478         if (!bg_cache) {
 479                 ASSERT(0);
 480                 return -ENOENT;
 481         }
 482
 483         /* Profiles not based on parity don't need full stripe lock */
 484         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
 485                 goto out;
 486         locks_root = &bg_cache->full_stripe_locks_root;
 487
 488         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
 489
 490         /* Now insert the full stripe lock */
 491         mutex_lock(&locks_root->lock);
 492         existing = insert_full_stripe_lock(locks_root, fstripe_start);
 493         mutex_unlock(&locks_root->lock);
 494         if (IS_ERR(existing)) {
 495                 ret = PTR_ERR(existing);
 496                 goto out;
 497         }
 498         mutex_lock(&existing->mutex);
 499         *locked_ret = true;
 500 out:
 501         btrfs_put_block_group(bg_cache);
 502         return ret;
 503 }
 504
 505 /*
 506  * Unlock a full stripe.
 507  *
 508  * NOTE: Caller must ensure it's the same context calling corresponding
 509  * lock_full_stripe().
 510  *
 511  * Return 0 if we unlock full stripe without problem.
 512  * Return <0 for error
 513  */
 514 static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
 515                               bool locked)
 516 {
 517         struct btrfs_block_group_cache *bg_cache;
 518         struct btrfs_full_stripe_locks_tree *locks_root;
 519         struct full_stripe_lock *fstripe_lock;
 520         u64 fstripe_start;
 521         bool freeit = false;
 522         int ret = 0;
 523
 524         /* If we didn't acquire full stripe lock, no need to continue */
 525         if (!locked)
 526                 return 0;
 527
 528         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
 529         if (!bg_cache) {
 530                 ASSERT(0);
 531                 return -ENOENT;
 532         }
 533         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
 534                 goto out;
 535
 536         locks_root = &bg_cache->full_stripe_locks_root;
 537         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
 538
 539         mutex_lock(&locks_root->lock);
 540         fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
 541         /* Unpaired unlock_full_stripe() detected */
 542         if (!fstripe_lock) {
 543                 WARN_ON(1);
 544                 ret = -ENOENT;
 545                 mutex_unlock(&locks_root->lock);
 546                 goto out;
 547         }
 548
 549         if (fstripe_lock->refs == 0) {
 550                 WARN_ON(1);
 551                 btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
 552                         fstripe_lock->logical);
 553         } else {
 554                 fstripe_lock->refs--;
 555         }
 556
 557         if (fstripe_lock->refs == 0) {
 558                 rb_erase(&fstripe_lock->node, &locks_root->root);
 559                 freeit = true;
 560         }
 561         mutex_unlock(&locks_root->lock);
 562
 563         mutex_unlock(&fstripe_lock->mutex);
 564         if (freeit)
 565                 kfree(fstripe_lock);
 566 out:
 567         btrfs_put_block_group(bg_cache);
 568         return ret;
 569 }
 570
 571 /*
 572  * used for workers that require transaction commits (i.e., for the
 573  * NOCOW case)
 574  */
 575 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 576 {
 577         struct btrfs_fs_info *fs_info = sctx->fs_info;
 578
 579         refcount_inc(&sctx->refs);
 580         /*
 581          * increment scrubs_running to prevent cancel requests from
 582          * completing as long as a worker is running. we must also
 583          * increment scrubs_paused to prevent deadlocking on pause
 584          * requests used for transactions commits (as the worker uses a
 585          * transaction context). it is safe to regard the worker
 586          * as paused for all matters practical. effectively, we only
 587          * avoid cancellation requests from completing.
 588          */
 589         mutex_lock(&fs_info->scrub_lock);
 590         atomic_inc(&fs_info->scrubs_running);
 591         atomic_inc(&fs_info->scrubs_paused);
 592         mutex_unlock(&fs_info->scrub_lock);
 593
 594         /*
 595          * check if @scrubs_running=@scrubs_paused condition
 596          * inside wait_event() is not an atomic operation.
 597          * which means we may inc/dec @scrub_running/paused
 598          * at any time. Let's wake up @scrub_pause_wait as
 599          * much as we can to let commit transaction blocked less.
 600          */
 601         wake_up(&fs_info->scrub_pause_wait);
 602
 603         atomic_inc(&sctx->workers_pending);
 604 }
 605
 606 /* used for workers that require transaction commits */
 607 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 608 {
 609         struct btrfs_fs_info *fs_info = sctx->fs_info;
 610
 611         /*
 612          * see scrub_pending_trans_workers_inc() why we're pretending
 613          * to be paused in the scrub counters
 614          */
 615         mutex_lock(&fs_info->scrub_lock);
 616         atomic_dec(&fs_info->scrubs_running);
 617         atomic_dec(&fs_info->scrubs_paused);
 618         mutex_unlock(&fs_info->scrub_lock);
 619         atomic_dec(&sctx->workers_pending);
 620         wake_up(&fs_info->scrub_pause_wait);
 621         wake_up(&sctx->list_wait);
 622         scrub_put_ctx(sctx);
 623 }
 624
 625 static void scrub_free_csums(struct scrub_ctx *sctx)
 626 {
 627         while (!list_empty(&sctx->csum_list)) {
 628                 struct btrfs_ordered_sum *sum;
 629                 sum = list_first_entry(&sctx->csum_list,
 630                                        struct btrfs_ordered_sum, list);
 631                 list_del(&sum->list);
 632                 kfree(sum);
 633         }
 634 }
 635
 636 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 637 {
 638         int i;
 639
 640         if (!sctx)
 641                 return;
 642
 643         /* this can happen when scrub is cancelled */
 644         if (sctx->curr != -1) {
 645                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 646
 647                 for (i = 0; i < sbio->page_count; i++) {
 648                         WARN_ON(!sbio->pagev[i]->page);
 649                         scrub_block_put(sbio->pagev[i]->sblock);
 650                 }
 651                 bio_put(sbio->bio);
 652         }
 653
 654         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 655                 struct scrub_bio *sbio = sctx->bios[i];
 656
 657                 if (!sbio)
 658                         break;
 659                 kfree(sbio);
 660         }
 661
 662         kfree(sctx->wr_curr_bio);
 663         scrub_free_csums(sctx);
 664         kfree(sctx);
 665 }
 666
 667 static void scrub_put_ctx(struct scrub_ctx *sctx)
 668 {
 669         if (refcount_dec_and_test(&sctx->refs))
 670                 scrub_free_ctx(sctx);
 671 }
 672
 673 static noinline_for_stack
 674 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 675 {
 676         struct scrub_ctx *sctx;
 677         int             i;
 678         struct btrfs_fs_info *fs_info = dev->fs_info;
 679
 680         sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
 681         if (!sctx)
 682                 goto nomem;
 683         refcount_set(&sctx->refs, 1);
 684         sctx->is_dev_replace = is_dev_replace;
 685         sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 686         sctx->curr = -1;
 687         sctx->fs_info = dev->fs_info;
 688         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 689                 struct scrub_bio *sbio;
 690
 691                 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
 692                 if (!sbio)
 693                         goto nomem;
 694                 sctx->bios[i] = sbio;
 695
 696                 sbio->index = i;
 697                 sbio->sctx = sctx;
 698                 sbio->page_count = 0;
 699                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
 700                                 scrub_bio_end_io_worker, NULL, NULL);
 701
 702                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 703                         sctx->bios[i]->next_free = i + 1;
 704                 else
 705                         sctx->bios[i]->next_free = -1;
 706         }
 707         sctx->first_free = 0;
 708         atomic_set(&sctx->bios_in_flight, 0);
 709         atomic_set(&sctx->workers_pending, 0);
 710         atomic_set(&sctx->cancel_req, 0);
 711         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 712         INIT_LIST_HEAD(&sctx->csum_list);
 713
 714         spin_lock_init(&sctx->list_lock);
 715         spin_lock_init(&sctx->stat_lock);
 716         init_waitqueue_head(&sctx->list_wait);
 717
 718         WARN_ON(sctx->wr_curr_bio != NULL);
 719         mutex_init(&sctx->wr_lock);
 720         sctx->wr_curr_bio = NULL;
 721         if (is_dev_replace) {
 722                 WARN_ON(!fs_info->dev_replace.tgtdev);
 723                 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
 724                 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
 725                 sctx->flush_all_writes = false;
 726         }
 727
 728         return sctx;
 729
 730 nomem:
 731         scrub_free_ctx(sctx);
 732         return ERR_PTR(-ENOMEM);
 733 }
 734
 735 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 736                                      void *warn_ctx)
 737 {
 738         u64 isize;
 739         u32 nlink;
 740         int ret;
 741         int i;
 742         unsigned nofs_flag;
 743         struct extent_buffer *eb;
 744         struct btrfs_inode_item *inode_item;
 745         struct scrub_warning *swarn = warn_ctx;
 746         struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
 747         struct inode_fs_paths *ipath = NULL;
 748         struct btrfs_root *local_root;
 749         struct btrfs_key root_key;
 750         struct btrfs_key key;
 751
 752         root_key.objectid = root;
 753         root_key.type = BTRFS_ROOT_ITEM_KEY;
 754         root_key.offset = (u64)-1;
 755         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 756         if (IS_ERR(local_root)) {
 757                 ret = PTR_ERR(local_root);
 758                 goto err;
 759         }
 760
 761         /*
 762          * this makes the path point to (inum INODE_ITEM ioff)
 763          */
 764         key.objectid = inum;
 765         key.type = BTRFS_INODE_ITEM_KEY;
 766         key.offset = 0;
 767
 768         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
 769         if (ret) {
 770                 btrfs_release_path(swarn->path);
 771                 goto err;
 772         }
 773
 774         eb = swarn->path->nodes[0];
 775         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 776                                         struct btrfs_inode_item);
 777         isize = btrfs_inode_size(eb, inode_item);
 778         nlink = btrfs_inode_nlink(eb, inode_item);
 779         btrfs_release_path(swarn->path);
 780
 781         /*
 782          * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
 783          * uses GFP_NOFS in this context, so we keep it consistent but it does
 784          * not seem to be strictly necessary.
 785          */
 786         nofs_flag = memalloc_nofs_save();
 787         ipath = init_ipath(4096, local_root, swarn->path);
 788         memalloc_nofs_restore(nofs_flag);
 789         if (IS_ERR(ipath)) {
 790                 ret = PTR_ERR(ipath);
 791                 ipath = NULL;
 792                 goto err;
 793         }
 794         ret = paths_from_inode(inum, ipath);
 795
 796         if (ret < 0)
 797                 goto err;
 798
 799         /*
 800          * we deliberately ignore the bit ipath might have been too small to
 801          * hold all of the paths here
 802          */
 803         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 804                 btrfs_warn_in_rcu(fs_info,
 805 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
 806                                   swarn->errstr, swarn->logical,
 807                                   rcu_str_deref(swarn->dev->name),
 808                                   swarn->physical,
 809                                   root, inum, offset,
 810                                   min(isize - offset, (u64)PAGE_SIZE), nlink,
 811                                   (char *)(unsigned long)ipath->fspath->val[i]);
 812
 813         free_ipath(ipath);
 814         return 0;
 815
 816 err:
 817         btrfs_warn_in_rcu(fs_info,
 818                           "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
 819                           swarn->errstr, swarn->logical,
 820                           rcu_str_deref(swarn->dev->name),
 821                           swarn->physical,
 822                           root, inum, offset, ret);
 823
 824         free_ipath(ipath);
 825         return 0;
 826 }
 827
 828 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 829 {
 830         struct btrfs_device *dev;
 831         struct btrfs_fs_info *fs_info;
 832         struct btrfs_path *path;
 833         struct btrfs_key found_key;
 834         struct extent_buffer *eb;
 835         struct btrfs_extent_item *ei;
 836         struct scrub_warning swarn;
 837         unsigned long ptr = 0;
 838         u64 extent_item_pos;
 839         u64 flags = 0;
 840         u64 ref_root;
 841         u32 item_size;
 842         u8 ref_level = 0;
 843         int ret;
 844
 845         WARN_ON(sblock->page_count < 1);
 846         dev = sblock->pagev[0]->dev;
 847         fs_info = sblock->sctx->fs_info;
 848
 849         path = btrfs_alloc_path();
 850         if (!path)
 851                 return;
 852
 853         swarn.physical = sblock->pagev[0]->physical;
 854         swarn.logical = sblock->pagev[0]->logical;
 855         swarn.errstr = errstr;
 856         swarn.dev = NULL;
 857
 858         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 859                                   &flags);
 860         if (ret < 0)
 861                 goto out;
 862
 863         extent_item_pos = swarn.logical - found_key.objectid;
 864         swarn.extent_item_size = found_key.offset;
 865
 866         eb = path->nodes[0];
 867         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 868         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 869
 870         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 871                 do {
 872                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
 873                                                       item_size, &ref_root,
 874                                                       &ref_level);
 875                         btrfs_warn_in_rcu(fs_info,
 876 "%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
 877                                 errstr, swarn.logical,
 878                                 rcu_str_deref(dev->name),
 879                                 swarn.physical,
 880                                 ref_level ? "node" : "leaf",
 881                                 ret < 0 ? -1 : ref_level,
 882                                 ret < 0 ? -1 : ref_root);
 883                 } while (ret != 1);
 884                 btrfs_release_path(path);
 885         } else {
 886                 btrfs_release_path(path);
 887                 swarn.path = path;
 888                 swarn.dev = dev;
 889                 iterate_extent_inodes(fs_info, found_key.objectid,
 890                                         extent_item_pos, 1,
 891                                         scrub_print_warning_inode, &swarn, false);
 892         }
 893
 894 out:
 895         btrfs_free_path(path);
 896 }
 897
 898 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 899 {
 900         struct page *page = NULL;
 901         unsigned long index;
 902         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 903         int ret;
 904         int corrected = 0;
 905         struct btrfs_key key;
 906         struct inode *inode = NULL;
 907         struct btrfs_fs_info *fs_info;
 908         u64 end = offset + PAGE_SIZE - 1;
 909         struct btrfs_root *local_root;
 910         int srcu_index;
 911
 912         key.objectid = root;
 913         key.type = BTRFS_ROOT_ITEM_KEY;
 914         key.offset = (u64)-1;
 915
 916         fs_info = fixup->root->fs_info;
 917         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
 918
 919         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
 920         if (IS_ERR(local_root)) {
 921                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 922                 return PTR_ERR(local_root);
 923         }
 924
 925         key.type = BTRFS_INODE_ITEM_KEY;
 926         key.objectid = inum;
 927         key.offset = 0;
 928         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
 929         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 930         if (IS_ERR(inode))
 931                 return PTR_ERR(inode);
 932
 933         index = offset >> PAGE_SHIFT;
 934
 935         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 936         if (!page) {
 937                 ret = -ENOMEM;
 938                 goto out;
 939         }
 940
 941         if (PageUptodate(page)) {
 942                 if (PageDirty(page)) {
 943                         /*
 944                          * we need to write the data to the defect sector. the
 945                          * data that was in that sector is not in memory,
 946                          * because the page was modified. we must not write the
 947                          * modified page to that sector.
 948                          *
 949                          * TODO: what could be done here: wait for the delalloc
 950                          *       runner to write out that page (might involve
 951                          *       COW) and see whether the sector is still
 952                          *       referenced afterwards.
 953                          *
 954                          * For the meantime, we'll treat this error
 955                          * incorrectable, although there is a chance that a
 956                          * later scrub will find the bad sector again and that
 957                          * there's no dirty page in memory, then.
 958                          */
 959                         ret = -EIO;
 960                         goto out;
 961                 }
 962                 ret = repair_io_failure(fs_info, inum, offset, PAGE_SIZE,
 963                                         fixup->logical, page,
 964                                         offset - page_offset(page),
 965                                         fixup->mirror_num);
 966                 unlock_page(page);
 967                 corrected = !ret;
 968         } else {
 969                 /*
 970                  * we need to get good data first. the general readpage path
 971                  * will call repair_io_failure for us, we just have to make
 972                  * sure we read the bad mirror.
 973                  */
 974                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 975                                         EXTENT_DAMAGED);
 976                 if (ret) {
 977                         /* set_extent_bits should give proper error */
 978                         WARN_ON(ret > 0);
 979                         if (ret > 0)
 980                                 ret = -EFAULT;
 981                         goto out;
 982                 }
 983
 984                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 985                                                 btrfs_get_extent,
 986                                                 fixup->mirror_num);
 987                 wait_on_page_locked(page);
 988
 989                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 990                                                 end, EXTENT_DAMAGED, 0, NULL);
 991                 if (!corrected)
 992                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 993                                                 EXTENT_DAMAGED);
 994         }
 995
 996 out:
 997         if (page)
 998                 put_page(page);
 999
1000         iput(inode);
1001
1002         if (ret < 0)
1003                 return ret;
1004
1005         if (ret == 0 && corrected) {
1006                 /*
1007                  * we only need to call readpage for one of the inodes belonging
1008                  * to this extent. so make iterate_extent_inodes stop
1009                  */
1010                 return 1;
1011         }
1012
1013         return -EIO;
1014 }
1015
1016 static void scrub_fixup_nodatasum(struct btrfs_work *work)
1017 {
1018         struct btrfs_fs_info *fs_info;
1019         int ret;
1020         struct scrub_fixup_nodatasum *fixup;
1021         struct scrub_ctx *sctx;
1022         struct btrfs_trans_handle *trans = NULL;
1023         struct btrfs_path *path;
1024         int uncorrectable = 0;
1025
1026         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
1027         sctx = fixup->sctx;
1028         fs_info = fixup->root->fs_info;
1029
1030         path = btrfs_alloc_path();
1031         if (!path) {
1032                 spin_lock(&sctx->stat_lock);
1033                 ++sctx->stat.malloc_errors;
1034                 spin_unlock(&sctx->stat_lock);
1035                 uncorrectable = 1;
1036                 goto out;
1037         }
1038
1039         trans = btrfs_join_transaction(fixup->root);
1040         if (IS_ERR(trans)) {
1041                 uncorrectable = 1;
1042                 goto out;
1043         }
1044
1045         /*
1046          * the idea is to trigger a regular read through the standard path. we
1047          * read a page from the (failed) logical address by specifying the
1048          * corresponding copynum of the failed sector. thus, that readpage is
1049          * expected to fail.
1050          * that is the point where on-the-fly error correction will kick in
1051          * (once it's finished) and rewrite the failed sector if a good copy
1052          * can be found.
1053          */
1054         ret = iterate_inodes_from_logical(fixup->logical, fs_info, path,
1055                                           scrub_fixup_readpage, fixup, false);
1056         if (ret < 0) {
1057                 uncorrectable = 1;
1058                 goto out;
1059         }
1060         WARN_ON(ret != 1);
1061
1062         spin_lock(&sctx->stat_lock);
1063         ++sctx->stat.corrected_errors;
1064         spin_unlock(&sctx->stat_lock);
1065
1066 out:
1067         if (trans && !IS_ERR(trans))
1068                 btrfs_end_transaction(trans);
1069         if (uncorrectable) {
1070                 spin_lock(&sctx->stat_lock);
1071                 ++sctx->stat.uncorrectable_errors;
1072                 spin_unlock(&sctx->stat_lock);
1073                 btrfs_dev_replace_stats_inc(
1074                         &fs_info->dev_replace.num_uncorrectable_read_errors);
1075                 btrfs_err_rl_in_rcu(fs_info,
1076                     "unable to fixup (nodatasum) error at logical %llu on dev %s",
1077                         fixup->logical, rcu_str_deref(fixup->dev->name));
1078         }
1079
1080         btrfs_free_path(path);
1081         kfree(fixup);
1082
1083         scrub_pending_trans_workers_dec(sctx);
1084 }
1085
1086 static inline void scrub_get_recover(struct scrub_recover *recover)
1087 {
1088         refcount_inc(&recover->refs);
1089 }
1090
1091 static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
1092                                      struct scrub_recover *recover)
1093 {
1094         if (refcount_dec_and_test(&recover->refs)) {
1095                 btrfs_bio_counter_dec(fs_info);
1096                 btrfs_put_bbio(recover->bbio);
1097                 kfree(recover);
1098         }
1099 }
1100
1101 /*
1102  * scrub_handle_errored_block gets called when either verification of the
1103  * pages failed or the bio failed to read, e.g. with EIO. In the latter
1104  * case, this function handles all pages in the bio, even though only one
1105  * may be bad.
1106  * The goal of this function is to repair the errored block by using the
1107  * contents of one of the mirrors.
1108  */
1109 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
1110 {
1111         struct scrub_ctx *sctx = sblock_to_check->sctx;
1112         struct btrfs_device *dev;
1113         struct btrfs_fs_info *fs_info;
1114         u64 logical;
1115         unsigned int failed_mirror_index;
1116         unsigned int is_metadata;
1117         unsigned int have_csum;
1118         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
1119         struct scrub_block *sblock_bad;
1120         int ret;
1121         int mirror_index;
1122         int page_num;
1123         int success;
1124         bool full_stripe_locked;
1125         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
1126                                       DEFAULT_RATELIMIT_BURST);
1127
1128         BUG_ON(sblock_to_check->page_count < 1);
1129         fs_info = sctx->fs_info;
1130         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
1131                 /*
1132                  * if we find an error in a super block, we just report it.
1133                  * They will get written with the next transaction commit
1134                  * anyway
1135                  */
1136                 spin_lock(&sctx->stat_lock);
1137                 ++sctx->stat.super_errors;
1138                 spin_unlock(&sctx->stat_lock);
1139                 return 0;
1140         }
1141         logical = sblock_to_check->pagev[0]->logical;
1142         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
1143         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
1144         is_metadata = !(sblock_to_check->pagev[0]->flags &
1145                         BTRFS_EXTENT_FLAG_DATA);
1146         have_csum = sblock_to_check->pagev[0]->have_csum;
1147         dev = sblock_to_check->pagev[0]->dev;
1148
1149         /*
1150          * For RAID5/6, race can happen for a different device scrub thread.
1151          * For data corruption, Parity and Data threads will both try
1152          * to recovery the data.
1153          * Race can lead to doubly added csum error, or even unrecoverable
1154          * error.
1155          */
1156         ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
1157         if (ret < 0) {
1158                 spin_lock(&sctx->stat_lock);
1159                 if (ret == -ENOMEM)
1160                         sctx->stat.malloc_errors++;
1161                 sctx->stat.read_errors++;
1162                 sctx->stat.uncorrectable_errors++;
1163                 spin_unlock(&sctx->stat_lock);
1164                 return ret;
1165         }
1166
1167         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
1168                 sblocks_for_recheck = NULL;
1169                 goto nodatasum_case;
1170         }
1171
1172         /*
1173          * read all mirrors one after the other. This includes to
1174          * re-read the extent or metadata block that failed (that was
1175          * the cause that this fixup code is called) another time,
1176          * page by page this time in order to know which pages
1177          * caused I/O errors and which ones are good (for all mirrors).
1178          * It is the goal to handle the situation when more than one
1179          * mirror contains I/O errors, but the errors do not
1180          * overlap, i.e. the data can be repaired by selecting the
1181          * pages from those mirrors without I/O error on the
1182          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
1183          * would be that mirror #1 has an I/O error on the first page,
1184          * the second page is good, and mirror #2 has an I/O error on
1185          * the second page, but the first page is good.
1186          * Then the first page of the first mirror can be repaired by
1187          * taking the first page of the second mirror, and the
1188          * second page of the second mirror can be repaired by
1189          * copying the contents of the 2nd page of the 1st mirror.
1190          * One more note: if the pages of one mirror contain I/O
1191          * errors, the checksum cannot be verified. In order to get
1192          * the best data for repairing, the first attempt is to find
1193          * a mirror without I/O errors and with a validated checksum.
1194          * Only if this is not possible, the pages are picked from
1195          * mirrors with I/O errors without considering the checksum.
1196          * If the latter is the case, at the end, the checksum of the
1197          * repaired area is verified in order to correctly maintain
1198          * the statistics.
1199          */
1200
1201         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
1202                                       sizeof(*sblocks_for_recheck), GFP_NOFS);
1203         if (!sblocks_for_recheck) {
1204                 spin_lock(&sctx->stat_lock);
1205                 sctx->stat.malloc_errors++;
1206                 sctx->stat.read_errors++;
1207                 sctx->stat.uncorrectable_errors++;
1208                 spin_unlock(&sctx->stat_lock);
1209                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1210                 goto out;
1211         }
1212
1213         /* setup the context, map the logical blocks and alloc the pages */
1214         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
1215         if (ret) {
1216                 spin_lock(&sctx->stat_lock);
1217                 sctx->stat.read_errors++;
1218                 sctx->stat.uncorrectable_errors++;
1219                 spin_unlock(&sctx->stat_lock);
1220                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1221                 goto out;
1222         }
1223         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
1224         sblock_bad = sblocks_for_recheck + failed_mirror_index;
1225
1226         /* build and submit the bios for the failed mirror, check checksums */
1227         scrub_recheck_block(fs_info, sblock_bad, 1);
1228
1229         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
1230             sblock_bad->no_io_error_seen) {
1231                 /*
1232                  * the error disappeared after reading page by page, or
1233                  * the area was part of a huge bio and other parts of the
1234                  * bio caused I/O errors, or the block layer merged several
1235                  * read requests into one and the error is caused by a
1236                  * different bio (usually one of the two latter cases is
1237                  * the cause)
1238                  */
1239                 spin_lock(&sctx->stat_lock);
1240                 sctx->stat.unverified_errors++;
1241                 sblock_to_check->data_corrected = 1;
1242                 spin_unlock(&sctx->stat_lock);
1243
1244                 if (sctx->is_dev_replace)
1245                         scrub_write_block_to_dev_replace(sblock_bad);
1246                 goto out;
1247         }
1248
1249         if (!sblock_bad->no_io_error_seen) {
1250                 spin_lock(&sctx->stat_lock);
1251                 sctx->stat.read_errors++;
1252                 spin_unlock(&sctx->stat_lock);
1253                 if (__ratelimit(&_rs))
1254                         scrub_print_warning("i/o error", sblock_to_check);
1255                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1256         } else if (sblock_bad->checksum_error) {
1257                 spin_lock(&sctx->stat_lock);
1258                 sctx->stat.csum_errors++;
1259                 spin_unlock(&sctx->stat_lock);
1260                 if (__ratelimit(&_rs))
1261                         scrub_print_warning("checksum error", sblock_to_check);
1262                 btrfs_dev_stat_inc_and_print(dev,
1263                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
1264         } else if (sblock_bad->header_error) {
1265                 spin_lock(&sctx->stat_lock);
1266                 sctx->stat.verify_errors++;
1267                 spin_unlock(&sctx->stat_lock);
1268                 if (__ratelimit(&_rs))
1269                         scrub_print_warning("checksum/header error",
1270                                             sblock_to_check);
1271                 if (sblock_bad->generation_error)
1272                         btrfs_dev_stat_inc_and_print(dev,
1273                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1274                 else
1275                         btrfs_dev_stat_inc_and_print(dev,
1276                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1277         }
1278
1279         if (sctx->readonly) {
1280                 ASSERT(!sctx->is_dev_replace);
1281                 goto out;
1282         }
1283
1284         if (!is_metadata && !have_csum) {
1285                 struct scrub_fixup_nodatasum *fixup_nodatasum;
1286
1287                 WARN_ON(sctx->is_dev_replace);
1288
1289 nodatasum_case:
1290
1291                 /*
1292                  * !is_metadata and !have_csum, this means that the data
1293                  * might not be COWed, that it might be modified
1294                  * concurrently. The general strategy to work on the
1295                  * commit root does not help in the case when COW is not
1296                  * used.
1297                  */
1298                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1299                 if (!fixup_nodatasum)
1300                         goto did_not_correct_error;
1301                 fixup_nodatasum->sctx = sctx;
1302                 fixup_nodatasum->dev = dev;
1303                 fixup_nodatasum->logical = logical;
1304                 fixup_nodatasum->root = fs_info->extent_root;
1305                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1306                 scrub_pending_trans_workers_inc(sctx);
1307                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1308                                 scrub_fixup_nodatasum, NULL, NULL);
1309                 btrfs_queue_work(fs_info->scrub_workers,
1310                                  &fixup_nodatasum->work);
1311                 goto out;
1312         }
1313
1314         /*
1315          * now build and submit the bios for the other mirrors, check
1316          * checksums.
1317          * First try to pick the mirror which is completely without I/O
1318          * errors and also does not have a checksum error.
1319          * If one is found, and if a checksum is present, the full block
1320          * that is known to contain an error is rewritten. Afterwards
1321          * the block is known to be corrected.
1322          * If a mirror is found which is completely correct, and no
1323          * checksum is present, only those pages are rewritten that had
1324          * an I/O error in the block to be repaired, since it cannot be
1325          * determined, which copy of the other pages is better (and it
1326          * could happen otherwise that a correct page would be
1327          * overwritten by a bad one).
1328          */
1329         for (mirror_index = 0; ;mirror_index++) {
1330                 struct scrub_block *sblock_other;
1331
1332                 if (mirror_index == failed_mirror_index)
1333                         continue;
1334
1335                 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
1336                 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1337                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1338                                 break;
1339                         if (!sblocks_for_recheck[mirror_index].page_count)
1340                                 break;
1341
1342                         sblock_other = sblocks_for_recheck + mirror_index;
1343                 } else {
1344                         struct scrub_recover *r = sblock_bad->pagev[0]->recover;
1345                         int max_allowed = r->bbio->num_stripes -
1346                                                 r->bbio->num_tgtdevs;
1347
1348                         if (mirror_index >= max_allowed)
1349                                 break;
1350                         if (!sblocks_for_recheck[1].page_count)
1351                                 break;
1352
1353                         ASSERT(failed_mirror_index == 0);
1354                         sblock_other = sblocks_for_recheck + 1;
1355                         sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
1356                 }
1357
1358                 /* build and submit the bios, check checksums */
1359                 scrub_recheck_block(fs_info, sblock_other, 0);
1360
1361                 if (!sblock_other->header_error &&
1362                     !sblock_other->checksum_error &&
1363                     sblock_other->no_io_error_seen) {
1364                         if (sctx->is_dev_replace) {
1365                                 scrub_write_block_to_dev_replace(sblock_other);
1366                                 goto corrected_error;
1367                         } else {
1368                                 ret = scrub_repair_block_from_good_copy(
1369                                                 sblock_bad, sblock_other);
1370                                 if (!ret)
1371                                         goto corrected_error;
1372                         }
1373                 }
1374         }
1375
1376         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1377                 goto did_not_correct_error;
1378
1379         /*
1380          * In case of I/O errors in the area that is supposed to be
1381          * repaired, continue by picking good copies of those pages.
1382          * Select the good pages from mirrors to rewrite bad pages from
1383          * the area to fix. Afterwards verify the checksum of the block
1384          * that is supposed to be repaired. This verification step is
1385          * only done for the purpose of statistic counting and for the
1386          * final scrub report, whether errors remain.
1387          * A perfect algorithm could make use of the checksum and try
1388          * all possible combinations of pages from the different mirrors
1389          * until the checksum verification succeeds. For example, when
1390          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1391          * of mirror #2 is readable but the final checksum test fails,
1392          * then the 2nd page of mirror #3 could be tried, whether now
1393          * the final checksum succeeds. But this would be a rare
1394          * exception and is therefore not implemented. At least it is
1395          * avoided that the good copy is overwritten.
1396          * A more useful improvement would be to pick the sectors
1397          * without I/O error based on sector sizes (512 bytes on legacy
1398          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1399          * mirror could be repaired by taking 512 byte of a different
1400          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1401          * area are unreadable.
1402          */
1403         success = 1;
1404         for (page_num = 0; page_num < sblock_bad->page_count;
1405              page_num++) {
1406                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1407                 struct scrub_block *sblock_other = NULL;
1408
1409                 /* skip no-io-error page in scrub */
1410                 if (!page_bad->io_error && !sctx->is_dev_replace)
1411                         continue;
1412
1413                 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1414                         /*
1415                          * In case of dev replace, if raid56 rebuild process
1416                          * didn't work out correct data, then copy the content
1417                          * in sblock_bad to make sure target device is identical
1418                          * to source device, instead of writing garbage data in
1419                          * sblock_for_recheck array to target device.
1420                          */
1421                         sblock_other = NULL;
1422                 } else if (page_bad->io_error) {
1423                         /* try to find no-io-error page in mirrors */
1424                         for (mirror_index = 0;
1425                              mirror_index < BTRFS_MAX_MIRRORS &&
1426                              sblocks_for_recheck[mirror_index].page_count > 0;
1427                              mirror_index++) {
1428                                 if (!sblocks_for_recheck[mirror_index].
1429                                     pagev[page_num]->io_error) {
1430                                         sblock_other = sblocks_for_recheck +
1431                                                        mirror_index;
1432                                         break;
1433                                 }
1434                         }
1435                         if (!sblock_other)
1436                                 success = 0;
1437                 }
1438
1439                 if (sctx->is_dev_replace) {
1440                         /*
1441                          * did not find a mirror to fetch the page
1442                          * from. scrub_write_page_to_dev_replace()
1443                          * handles this case (page->io_error), by
1444                          * filling the block with zeros before
1445                          * submitting the write request
1446                          */
1447                         if (!sblock_other)
1448                                 sblock_other = sblock_bad;
1449
1450                         if (scrub_write_page_to_dev_replace(sblock_other,
1451                                                             page_num) != 0) {
1452                                 btrfs_dev_replace_stats_inc(
1453                                         &fs_info->dev_replace.num_write_errors);
1454                                 success = 0;
1455                         }
1456                 } else if (sblock_other) {
1457                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1458                                                                sblock_other,
1459                                                                page_num, 0);
1460                         if (0 == ret)
1461                                 page_bad->io_error = 0;
1462                         else
1463                                 success = 0;
1464                 }
1465         }
1466
1467         if (success && !sctx->is_dev_replace) {
1468                 if (is_metadata || have_csum) {
1469                         /*
1470                          * need to verify the checksum now that all
1471                          * sectors on disk are repaired (the write
1472                          * request for data to be repaired is on its way).
1473                          * Just be lazy and use scrub_recheck_block()
1474                          * which re-reads the data before the checksum
1475                          * is verified, but most likely the data comes out
1476                          * of the page cache.
1477                          */
1478                         scrub_recheck_block(fs_info, sblock_bad, 1);
1479                         if (!sblock_bad->header_error &&
1480                             !sblock_bad->checksum_error &&
1481                             sblock_bad->no_io_error_seen)
1482                                 goto corrected_error;
1483                         else
1484                                 goto did_not_correct_error;
1485                 } else {
1486 corrected_error:
1487                         spin_lock(&sctx->stat_lock);
1488                         sctx->stat.corrected_errors++;
1489                         sblock_to_check->data_corrected = 1;
1490                         spin_unlock(&sctx->stat_lock);
1491                         btrfs_err_rl_in_rcu(fs_info,
1492                                 "fixed up error at logical %llu on dev %s",
1493                                 logical, rcu_str_deref(dev->name));
1494                 }
1495         } else {
1496 did_not_correct_error:
1497                 spin_lock(&sctx->stat_lock);
1498                 sctx->stat.uncorrectable_errors++;
1499                 spin_unlock(&sctx->stat_lock);
1500                 btrfs_err_rl_in_rcu(fs_info,
1501                         "unable to fixup (regular) error at logical %llu on dev %s",
1502                         logical, rcu_str_deref(dev->name));
1503         }
1504
1505 out:
1506         if (sblocks_for_recheck) {
1507                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1508                      mirror_index++) {
1509                         struct scrub_block *sblock = sblocks_for_recheck +
1510                                                      mirror_index;
1511                         struct scrub_recover *recover;
1512                         int page_index;
1513
1514                         for (page_index = 0; page_index < sblock->page_count;
1515                              page_index++) {
1516                                 sblock->pagev[page_index]->sblock = NULL;
1517                                 recover = sblock->pagev[page_index]->recover;
1518                                 if (recover) {
1519                                         scrub_put_recover(fs_info, recover);
1520                                         sblock->pagev[page_index]->recover =
1521                                                                         NULL;
1522                                 }
1523                                 scrub_page_put(sblock->pagev[page_index]);
1524                         }
1525                 }
1526                 kfree(sblocks_for_recheck);
1527         }
1528
1529         ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
1530         if (ret < 0)
1531                 return ret;
1532         return 0;
1533 }
1534
1535 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1536 {
1537         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1538                 return 2;
1539         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1540                 return 3;
1541         else
1542                 return (int)bbio->num_stripes;
1543 }
1544
1545 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1546                                                  u64 *raid_map,
1547                                                  u64 mapped_length,
1548                                                  int nstripes, int mirror,
1549                                                  int *stripe_index,
1550                                                  u64 *stripe_offset)
1551 {
1552         int i;
1553
1554         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1555                 /* RAID5/6 */
1556                 for (i = 0; i < nstripes; i++) {
1557                         if (raid_map[i] == RAID6_Q_STRIPE ||
1558                             raid_map[i] == RAID5_P_STRIPE)
1559                                 continue;
1560
1561                         if (logical >= raid_map[i] &&
1562                             logical < raid_map[i] + mapped_length)
1563                                 break;
1564                 }
1565
1566                 *stripe_index = i;
1567                 *stripe_offset = logical - raid_map[i];
1568         } else {
1569                 /* The other RAID type */
1570                 *stripe_index = mirror;
1571                 *stripe_offset = 0;
1572         }
1573 }
1574
1575 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1576                                      struct scrub_block *sblocks_for_recheck)
1577 {
1578         struct scrub_ctx *sctx = original_sblock->sctx;
1579         struct btrfs_fs_info *fs_info = sctx->fs_info;
1580         u64 length = original_sblock->page_count * PAGE_SIZE;
1581         u64 logical = original_sblock->pagev[0]->logical;
1582         u64 generation = original_sblock->pagev[0]->generation;
1583         u64 flags = original_sblock->pagev[0]->flags;
1584         u64 have_csum = original_sblock->pagev[0]->have_csum;
1585         struct scrub_recover *recover;
1586         struct btrfs_bio *bbio;
1587         u64 sublen;
1588         u64 mapped_length;
1589         u64 stripe_offset;
1590         int stripe_index;
1591         int page_index = 0;
1592         int mirror_index;
1593         int nmirrors;
1594         int ret;
1595
1596         /*
1597          * note: the two members refs and outstanding_pages
1598          * are not used (and not set) in the blocks that are used for
1599          * the recheck procedure
1600          */
1601
1602         while (length > 0) {
1603                 sublen = min_t(u64, length, PAGE_SIZE);
1604                 mapped_length = sublen;
1605                 bbio = NULL;
1606
1607                 /*
1608                  * with a length of PAGE_SIZE, each returned stripe
1609                  * represents one mirror
1610                  */
1611                 btrfs_bio_counter_inc_blocked(fs_info);
1612                 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1613                                 logical, &mapped_length, &bbio);
1614                 if (ret || !bbio || mapped_length < sublen) {
1615                         btrfs_put_bbio(bbio);
1616                         btrfs_bio_counter_dec(fs_info);
1617                         return -EIO;
1618                 }
1619
1620                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1621                 if (!recover) {
1622                         btrfs_put_bbio(bbio);
1623                         btrfs_bio_counter_dec(fs_info);
1624                         return -ENOMEM;
1625                 }
1626
1627                 refcount_set(&recover->refs, 1);
1628                 recover->bbio = bbio;
1629                 recover->map_length = mapped_length;
1630
1631                 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1632
1633                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1634
1635                 for (mirror_index = 0; mirror_index < nmirrors;
1636                      mirror_index++) {
1637                         struct scrub_block *sblock;
1638                         struct scrub_page *page;
1639
1640                         sblock = sblocks_for_recheck + mirror_index;
1641                         sblock->sctx = sctx;
1642
1643                         page = kzalloc(sizeof(*page), GFP_NOFS);
1644                         if (!page) {
1645 leave_nomem:
1646                                 spin_lock(&sctx->stat_lock);
1647                                 sctx->stat.malloc_errors++;
1648                                 spin_unlock(&sctx->stat_lock);
1649                                 scrub_put_recover(fs_info, recover);
1650                                 return -ENOMEM;
1651                         }
1652                         scrub_page_get(page);
1653                         sblock->pagev[page_index] = page;
1654                         page->sblock = sblock;
1655                         page->flags = flags;
1656                         page->generation = generation;
1657                         page->logical = logical;
1658                         page->have_csum = have_csum;
1659                         if (have_csum)
1660                                 memcpy(page->csum,
1661                                        original_sblock->pagev[0]->csum,
1662                                        sctx->csum_size);
1663
1664                         scrub_stripe_index_and_offset(logical,
1665                                                       bbio->map_type,
1666                                                       bbio->raid_map,
1667                                                       mapped_length,
1668                                                       bbio->num_stripes -
1669                                                       bbio->num_tgtdevs,
1670                                                       mirror_index,
1671                                                       &stripe_index,
1672                                                       &stripe_offset);
1673                         page->physical = bbio->stripes[stripe_index].physical +
1674                                          stripe_offset;
1675                         page->dev = bbio->stripes[stripe_index].dev;
1676
1677                         BUG_ON(page_index >= original_sblock->page_count);
1678                         page->physical_for_dev_replace =
1679                                 original_sblock->pagev[page_index]->
1680                                 physical_for_dev_replace;
1681                         /* for missing devices, dev->bdev is NULL */
1682                         page->mirror_num = mirror_index + 1;
1683                         sblock->page_count++;
1684                         page->page = alloc_page(GFP_NOFS);
1685                         if (!page->page)
1686                                 goto leave_nomem;
1687
1688                         scrub_get_recover(recover);
1689                         page->recover = recover;
1690                 }
1691                 scrub_put_recover(fs_info, recover);
1692                 length -= sublen;
1693                 logical += sublen;
1694                 page_index++;
1695         }
1696
1697         return 0;
1698 }
1699
1700 static void scrub_bio_wait_endio(struct bio *bio)
1701 {
1702         complete(bio->bi_private);
1703 }
1704
1705 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1706                                         struct bio *bio,
1707                                         struct scrub_page *page)
1708 {
1709         DECLARE_COMPLETION_ONSTACK(done);
1710         int ret;
1711         int mirror_num;
1712
1713         bio->bi_iter.bi_sector = page->logical >> 9;
1714         bio->bi_private = &done;
1715         bio->bi_end_io = scrub_bio_wait_endio;
1716
1717         mirror_num = page->sblock->pagev[0]->mirror_num;
1718         ret = raid56_parity_recover(fs_info, bio, page->recover->bbio,
1719                                     page->recover->map_length,
1720                                     mirror_num, 0);
1721         if (ret)
1722                 return ret;
1723
1724         wait_for_completion_io(&done);
1725         return blk_status_to_errno(bio->bi_status);
1726 }
1727
1728 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
1729                                           struct scrub_block *sblock)
1730 {
1731         struct scrub_page *first_page = sblock->pagev[0];
1732         struct bio *bio;
1733         int page_num;
1734
1735         /* All pages in sblock belong to the same stripe on the same device. */
1736         ASSERT(first_page->dev);
1737         if (!first_page->dev->bdev)
1738                 goto out;
1739
1740         bio = btrfs_io_bio_alloc(BIO_MAX_PAGES);
1741         bio_set_dev(bio, first_page->dev->bdev);
1742
1743         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1744                 struct scrub_page *page = sblock->pagev[page_num];
1745
1746                 WARN_ON(!page->page);
1747                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1748         }
1749
1750         if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
1751                 bio_put(bio);
1752                 goto out;
1753         }
1754
1755         bio_put(bio);
1756
1757         scrub_recheck_block_checksum(sblock);
1758
1759         return;
1760 out:
1761         for (page_num = 0; page_num < sblock->page_count; page_num++)
1762                 sblock->pagev[page_num]->io_error = 1;
1763
1764         sblock->no_io_error_seen = 0;
1765 }
1766
1767 /*
1768  * this function will check the on disk data for checksum errors, header
1769  * errors and read I/O errors. If any I/O errors happen, the exact pages
1770  * which are errored are marked as being bad. The goal is to enable scrub
1771  * to take those pages that are not errored from all the mirrors so that
1772  * the pages that are errored in the just handled mirror can be repaired.
1773  */
1774 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1775                                 struct scrub_block *sblock,
1776                                 int retry_failed_mirror)
1777 {
1778         int page_num;
1779
1780         sblock->no_io_error_seen = 1;
1781
1782         /* short cut for raid56 */
1783         if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
1784                 return scrub_recheck_block_on_raid56(fs_info, sblock);
1785
1786         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1787                 struct bio *bio;
1788                 struct scrub_page *page = sblock->pagev[page_num];
1789
1790                 if (page->dev->bdev == NULL) {
1791                         page->io_error = 1;
1792                         sblock->no_io_error_seen = 0;
1793                         continue;
1794                 }
1795
1796                 WARN_ON(!page->page);
1797                 bio = btrfs_io_bio_alloc(1);
1798                 bio_set_dev(bio, page->dev->bdev);
1799
1800                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1801                 bio->bi_iter.bi_sector = page->physical >> 9;
1802                 bio->bi_opf = REQ_OP_READ;
1803
1804                 if (btrfsic_submit_bio_wait(bio)) {
1805                         page->io_error = 1;
1806                         sblock->no_io_error_seen = 0;
1807                 }
1808
1809                 bio_put(bio);
1810         }
1811
1812         if (sblock->no_io_error_seen)
1813                 scrub_recheck_block_checksum(sblock);
1814 }
1815
1816 static inline int scrub_check_fsid(u8 fsid[],
1817                                    struct scrub_page *spage)
1818 {
1819         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1820         int ret;
1821
1822         ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
1823         return !ret;
1824 }
1825
1826 static void scrub_recheck_block_checksum(struct scrub_block *sblock)
1827 {
1828         sblock->header_error = 0;
1829         sblock->checksum_error = 0;
1830         sblock->generation_error = 0;
1831
1832         if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
1833                 scrub_checksum_data(sblock);
1834         else
1835                 scrub_checksum_tree_block(sblock);
1836 }
1837
1838 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1839                                              struct scrub_block *sblock_good)
1840 {
1841         int page_num;
1842         int ret = 0;
1843
1844         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1845                 int ret_sub;
1846
1847                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1848                                                            sblock_good,
1849                                                            page_num, 1);
1850                 if (ret_sub)
1851                         ret = ret_sub;
1852         }
1853
1854         return ret;
1855 }
1856
1857 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1858                                             struct scrub_block *sblock_good,
1859                                             int page_num, int force_write)
1860 {
1861         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1862         struct scrub_page *page_good = sblock_good->pagev[page_num];
1863         struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
1864
1865         BUG_ON(page_bad->page == NULL);
1866         BUG_ON(page_good->page == NULL);
1867         if (force_write || sblock_bad->header_error ||
1868             sblock_bad->checksum_error || page_bad->io_error) {
1869                 struct bio *bio;
1870                 int ret;
1871
1872                 if (!page_bad->dev->bdev) {
1873                         btrfs_warn_rl(fs_info,
1874                                 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1875                         return -EIO;
1876                 }
1877
1878                 bio = btrfs_io_bio_alloc(1);
1879                 bio_set_dev(bio, page_bad->dev->bdev);
1880                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1881                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1882
1883                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1884                 if (PAGE_SIZE != ret) {
1885                         bio_put(bio);
1886                         return -EIO;
1887                 }
1888
1889                 if (btrfsic_submit_bio_wait(bio)) {
1890                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1891                                 BTRFS_DEV_STAT_WRITE_ERRS);
1892                         btrfs_dev_replace_stats_inc(
1893                                 &fs_info->dev_replace.num_write_errors);
1894                         bio_put(bio);
1895                         return -EIO;
1896                 }
1897                 bio_put(bio);
1898         }
1899
1900         return 0;
1901 }
1902
1903 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1904 {
1905         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
1906         int page_num;
1907
1908         /*
1909          * This block is used for the check of the parity on the source device,
1910          * so the data needn't be written into the destination device.
1911          */
1912         if (sblock->sparity)
1913                 return;
1914
1915         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1916                 int ret;
1917
1918                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1919                 if (ret)
1920                         btrfs_dev_replace_stats_inc(
1921                                 &fs_info->dev_replace.num_write_errors);
1922         }
1923 }
1924
1925 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1926                                            int page_num)
1927 {
1928         struct scrub_page *spage = sblock->pagev[page_num];
1929
1930         BUG_ON(spage->page == NULL);
1931         if (spage->io_error) {
1932                 void *mapped_buffer = kmap_atomic(spage->page);
1933
1934                 clear_page(mapped_buffer);
1935                 flush_dcache_page(spage->page);
1936                 kunmap_atomic(mapped_buffer);
1937         }
1938         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1939 }
1940
1941 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1942                                     struct scrub_page *spage)
1943 {
1944         struct scrub_bio *sbio;
1945         int ret;
1946
1947         mutex_lock(&sctx->wr_lock);
1948 again:
1949         if (!sctx->wr_curr_bio) {
1950                 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
1951                                               GFP_KERNEL);
1952                 if (!sctx->wr_curr_bio) {
1953                         mutex_unlock(&sctx->wr_lock);
1954                         return -ENOMEM;
1955                 }
1956                 sctx->wr_curr_bio->sctx = sctx;
1957                 sctx->wr_curr_bio->page_count = 0;
1958         }
1959         sbio = sctx->wr_curr_bio;
1960         if (sbio->page_count == 0) {
1961                 struct bio *bio;
1962
1963                 sbio->physical = spage->physical_for_dev_replace;
1964                 sbio->logical = spage->logical;
1965                 sbio->dev = sctx->wr_tgtdev;
1966                 bio = sbio->bio;
1967                 if (!bio) {
1968                         bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio);
1969                         sbio->bio = bio;
1970                 }
1971
1972                 bio->bi_private = sbio;
1973                 bio->bi_end_io = scrub_wr_bio_end_io;
1974                 bio_set_dev(bio, sbio->dev->bdev);
1975                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1976                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1977                 sbio->status = 0;
1978         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1979                    spage->physical_for_dev_replace ||
1980                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1981                    spage->logical) {
1982                 scrub_wr_submit(sctx);
1983                 goto again;
1984         }
1985
1986         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1987         if (ret != PAGE_SIZE) {
1988                 if (sbio->page_count < 1) {
1989                         bio_put(sbio->bio);
1990                         sbio->bio = NULL;
1991                         mutex_unlock(&sctx->wr_lock);
1992                         return -EIO;
1993                 }
1994                 scrub_wr_submit(sctx);
1995                 goto again;
1996         }
1997
1998         sbio->pagev[sbio->page_count] = spage;
1999         scrub_page_get(spage);
2000         sbio->page_count++;
2001         if (sbio->page_count == sctx->pages_per_wr_bio)
2002                 scrub_wr_submit(sctx);
2003         mutex_unlock(&sctx->wr_lock);
2004
2005         return 0;
2006 }
2007
2008 static void scrub_wr_submit(struct scrub_ctx *sctx)
2009 {
2010         struct scrub_bio *sbio;
2011
2012         if (!sctx->wr_curr_bio)
2013                 return;
2014
2015         sbio = sctx->wr_curr_bio;
2016         sctx->wr_curr_bio = NULL;
2017         WARN_ON(!sbio->bio->bi_disk);
2018         scrub_pending_bio_inc(sctx);
2019         /* process all writes in a single worker thread. Then the block layer
2020          * orders the requests before sending them to the driver which
2021          * doubled the write performance on spinning disks when measured
2022          * with Linux 3.5 */
2023         btrfsic_submit_bio(sbio->bio);
2024 }
2025
2026 static void scrub_wr_bio_end_io(struct bio *bio)
2027 {
2028         struct scrub_bio *sbio = bio->bi_private;
2029         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2030
2031         sbio->status = bio->bi_status;
2032         sbio->bio = bio;
2033
2034         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
2035                          scrub_wr_bio_end_io_worker, NULL, NULL);
2036         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
2037 }
2038
2039 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
2040 {
2041         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2042         struct scrub_ctx *sctx = sbio->sctx;
2043         int i;
2044
2045         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
2046         if (sbio->status) {
2047                 struct btrfs_dev_replace *dev_replace =
2048                         &sbio->sctx->fs_info->dev_replace;
2049
2050                 for (i = 0; i < sbio->page_count; i++) {
2051                         struct scrub_page *spage = sbio->pagev[i];
2052
2053                         spage->io_error = 1;
2054                         btrfs_dev_replace_stats_inc(&dev_replace->
2055                                                     num_write_errors);
2056                 }
2057         }
2058
2059         for (i = 0; i < sbio->page_count; i++)
2060                 scrub_page_put(sbio->pagev[i]);
2061
2062         bio_put(sbio->bio);
2063         kfree(sbio);
2064         scrub_pending_bio_dec(sctx);
2065 }
2066
2067 static int scrub_checksum(struct scrub_block *sblock)
2068 {
2069         u64 flags;
2070         int ret;
2071
2072         /*
2073          * No need to initialize these stats currently,
2074          * because this function only use return value
2075          * instead of these stats value.
2076          *
2077          * Todo:
2078          * always use stats
2079          */
2080         sblock->header_error = 0;
2081         sblock->generation_error = 0;
2082         sblock->checksum_error = 0;
2083
2084         WARN_ON(sblock->page_count < 1);
2085         flags = sblock->pagev[0]->flags;
2086         ret = 0;
2087         if (flags & BTRFS_EXTENT_FLAG_DATA)
2088                 ret = scrub_checksum_data(sblock);
2089         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
2090                 ret = scrub_checksum_tree_block(sblock);
2091         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
2092                 (void)scrub_checksum_super(sblock);
2093         else
2094                 WARN_ON(1);
2095         if (ret)
2096                 scrub_handle_errored_block(sblock);
2097
2098         return ret;
2099 }
2100
2101 static int scrub_checksum_data(struct scrub_block *sblock)
2102 {
2103         struct scrub_ctx *sctx = sblock->sctx;
2104         u8 csum[BTRFS_CSUM_SIZE];
2105         u8 *on_disk_csum;
2106         struct page *page;
2107         void *buffer;
2108         u32 crc = ~(u32)0;
2109         u64 len;
2110         int index;
2111
2112         BUG_ON(sblock->page_count < 1);
2113         if (!sblock->pagev[0]->have_csum)
2114                 return 0;
2115
2116         on_disk_csum = sblock->pagev[0]->csum;
2117         page = sblock->pagev[0]->page;
2118         buffer = kmap_atomic(page);
2119
2120         len = sctx->fs_info->sectorsize;
2121         index = 0;
2122         for (;;) {
2123                 u64 l = min_t(u64, len, PAGE_SIZE);
2124
2125                 crc = btrfs_csum_data(buffer, crc, l);
2126                 kunmap_atomic(buffer);
2127                 len -= l;
2128                 if (len == 0)
2129                         break;
2130                 index++;
2131                 BUG_ON(index >= sblock->page_count);
2132                 BUG_ON(!sblock->pagev[index]->page);
2133                 page = sblock->pagev[index]->page;
2134                 buffer = kmap_atomic(page);
2135         }
2136
2137         btrfs_csum_final(crc, csum);
2138         if (memcmp(csum, on_disk_csum, sctx->csum_size))
2139                 sblock->checksum_error = 1;
2140
2141         return sblock->checksum_error;
2142 }
2143
2144 static int scrub_checksum_tree_block(struct scrub_block *sblock)
2145 {
2146         struct scrub_ctx *sctx = sblock->sctx;
2147         struct btrfs_header *h;
2148         struct btrfs_fs_info *fs_info = sctx->fs_info;
2149         u8 calculated_csum[BTRFS_CSUM_SIZE];
2150         u8 on_disk_csum[BTRFS_CSUM_SIZE];
2151         struct page *page;
2152         void *mapped_buffer;
2153         u64 mapped_size;
2154         void *p;
2155         u32 crc = ~(u32)0;
2156         u64 len;
2157         int index;
2158
2159         BUG_ON(sblock->page_count < 1);
2160         page = sblock->pagev[0]->page;
2161         mapped_buffer = kmap_atomic(page);
2162         h = (struct btrfs_header *)mapped_buffer;
2163         memcpy(on_disk_csum, h->csum, sctx->csum_size);
2164
2165         /*
2166          * we don't use the getter functions here, as we
2167          * a) don't have an extent buffer and
2168          * b) the page is already kmapped
2169          */
2170         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
2171                 sblock->header_error = 1;
2172
2173         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {
2174                 sblock->header_error = 1;
2175                 sblock->generation_error = 1;
2176         }
2177
2178         if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
2179                 sblock->header_error = 1;
2180
2181         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
2182                    BTRFS_UUID_SIZE))
2183                 sblock->header_error = 1;
2184
2185         len = sctx->fs_info->nodesize - BTRFS_CSUM_SIZE;
2186         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2187         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2188         index = 0;
2189         for (;;) {
2190                 u64 l = min_t(u64, len, mapped_size);
2191
2192                 crc = btrfs_csum_data(p, crc, l);
2193                 kunmap_atomic(mapped_buffer);
2194                 len -= l;
2195                 if (len == 0)
2196                         break;
2197                 index++;
2198                 BUG_ON(index >= sblock->page_count);
2199                 BUG_ON(!sblock->pagev[index]->page);
2200                 page = sblock->pagev[index]->page;
2201                 mapped_buffer = kmap_atomic(page);
2202                 mapped_size = PAGE_SIZE;
2203                 p = mapped_buffer;
2204         }
2205
2206         btrfs_csum_final(crc, calculated_csum);
2207         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2208                 sblock->checksum_error = 1;
2209
2210         return sblock->header_error || sblock->checksum_error;
2211 }
2212
2213 static int scrub_checksum_super(struct scrub_block *sblock)
2214 {
2215         struct btrfs_super_block *s;
2216         struct scrub_ctx *sctx = sblock->sctx;
2217         u8 calculated_csum[BTRFS_CSUM_SIZE];
2218         u8 on_disk_csum[BTRFS_CSUM_SIZE];
2219         struct page *page;
2220         void *mapped_buffer;
2221         u64 mapped_size;
2222         void *p;
2223         u32 crc = ~(u32)0;
2224         int fail_gen = 0;
2225         int fail_cor = 0;
2226         u64 len;
2227         int index;
2228
2229         BUG_ON(sblock->page_count < 1);
2230         page = sblock->pagev[0]->page;
2231         mapped_buffer = kmap_atomic(page);
2232         s = (struct btrfs_super_block *)mapped_buffer;
2233         memcpy(on_disk_csum, s->csum, sctx->csum_size);
2234
2235         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2236                 ++fail_cor;
2237
2238         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2239                 ++fail_gen;
2240
2241         if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2242                 ++fail_cor;
2243
2244         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2245         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2246         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2247         index = 0;
2248         for (;;) {
2249                 u64 l = min_t(u64, len, mapped_size);
2250
2251                 crc = btrfs_csum_data(p, crc, l);
2252                 kunmap_atomic(mapped_buffer);
2253                 len -= l;
2254                 if (len == 0)
2255                         break;
2256                 index++;
2257                 BUG_ON(index >= sblock->page_count);
2258                 BUG_ON(!sblock->pagev[index]->page);
2259                 page = sblock->pagev[index]->page;
2260                 mapped_buffer = kmap_atomic(page);
2261                 mapped_size = PAGE_SIZE;
2262                 p = mapped_buffer;
2263         }
2264
2265         btrfs_csum_final(crc, calculated_csum);
2266         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2267                 ++fail_cor;
2268
2269         if (fail_cor + fail_gen) {
2270                 /*
2271                  * if we find an error in a super block, we just report it.
2272                  * They will get written with the next transaction commit
2273                  * anyway
2274                  */
2275                 spin_lock(&sctx->stat_lock);
2276                 ++sctx->stat.super_errors;
2277                 spin_unlock(&sctx->stat_lock);
2278                 if (fail_cor)
2279                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2280                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2281                 else
2282                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2283                                 BTRFS_DEV_STAT_GENERATION_ERRS);
2284         }
2285
2286         return fail_cor + fail_gen;
2287 }
2288
2289 static void scrub_block_get(struct scrub_block *sblock)
2290 {
2291         refcount_inc(&sblock->refs);
2292 }
2293
2294 static void scrub_block_put(struct scrub_block *sblock)
2295 {
2296         if (refcount_dec_and_test(&sblock->refs)) {
2297                 int i;
2298
2299                 if (sblock->sparity)
2300                         scrub_parity_put(sblock->sparity);
2301
2302                 for (i = 0; i < sblock->page_count; i++)
2303                         scrub_page_put(sblock->pagev[i]);
2304                 kfree(sblock);
2305         }
2306 }
2307
2308 static void scrub_page_get(struct scrub_page *spage)
2309 {
2310         atomic_inc(&spage->refs);
2311 }
2312
2313 static void scrub_page_put(struct scrub_page *spage)
2314 {
2315         if (atomic_dec_and_test(&spage->refs)) {
2316                 if (spage->page)
2317                         __free_page(spage->page);
2318                 kfree(spage);
2319         }
2320 }
2321
2322 static void scrub_submit(struct scrub_ctx *sctx)
2323 {
2324         struct scrub_bio *sbio;
2325
2326         if (sctx->curr == -1)
2327                 return;
2328
2329         sbio = sctx->bios[sctx->curr];
2330         sctx->curr = -1;
2331         scrub_pending_bio_inc(sctx);
2332         btrfsic_submit_bio(sbio->bio);
2333 }
2334
2335 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2336                                     struct scrub_page *spage)
2337 {
2338         struct scrub_block *sblock = spage->sblock;
2339         struct scrub_bio *sbio;
2340         int ret;
2341
2342 again:
2343         /*
2344          * grab a fresh bio or wait for one to become available
2345          */
2346         while (sctx->curr == -1) {
2347                 spin_lock(&sctx->list_lock);
2348                 sctx->curr = sctx->first_free;
2349                 if (sctx->curr != -1) {
2350                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
2351                         sctx->bios[sctx->curr]->next_free = -1;
2352                         sctx->bios[sctx->curr]->page_count = 0;
2353                         spin_unlock(&sctx->list_lock);
2354                 } else {
2355                         spin_unlock(&sctx->list_lock);
2356                         wait_event(sctx->list_wait, sctx->first_free != -1);
2357                 }
2358         }
2359         sbio = sctx->bios[sctx->curr];
2360         if (sbio->page_count == 0) {
2361                 struct bio *bio;
2362
2363                 sbio->physical = spage->physical;
2364                 sbio->logical = spage->logical;
2365                 sbio->dev = spage->dev;
2366                 bio = sbio->bio;
2367                 if (!bio) {
2368                         bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio);
2369                         sbio->bio = bio;
2370                 }
2371
2372                 bio->bi_private = sbio;
2373                 bio->bi_end_io = scrub_bio_end_io;
2374                 bio_set_dev(bio, sbio->dev->bdev);
2375                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2376                 bio_set_op_attrs(bio, REQ_OP_READ, 0);
2377                 sbio->status = 0;
2378         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2379                    spage->physical ||
2380                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2381                    spage->logical ||
2382                    sbio->dev != spage->dev) {
2383                 scrub_submit(sctx);
2384                 goto again;
2385         }
2386
2387         sbio->pagev[sbio->page_count] = spage;
2388         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2389         if (ret != PAGE_SIZE) {
2390                 if (sbio->page_count < 1) {
2391                         bio_put(sbio->bio);
2392                         sbio->bio = NULL;
2393                         return -EIO;
2394                 }
2395                 scrub_submit(sctx);
2396                 goto again;
2397         }
2398
2399         scrub_block_get(sblock); /* one for the page added to the bio */
2400         atomic_inc(&sblock->outstanding_pages);
2401         sbio->page_count++;
2402         if (sbio->page_count == sctx->pages_per_rd_bio)
2403                 scrub_submit(sctx);
2404
2405         return 0;
2406 }
2407
2408 static void scrub_missing_raid56_end_io(struct bio *bio)
2409 {
2410         struct scrub_block *sblock = bio->bi_private;
2411         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
2412
2413         if (bio->bi_status)
2414                 sblock->no_io_error_seen = 0;
2415
2416         bio_put(bio);
2417
2418         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2419 }
2420
2421 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2422 {
2423         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2424         struct scrub_ctx *sctx = sblock->sctx;
2425         struct btrfs_fs_info *fs_info = sctx->fs_info;
2426         u64 logical;
2427         struct btrfs_device *dev;
2428
2429         logical = sblock->pagev[0]->logical;
2430         dev = sblock->pagev[0]->dev;
2431
2432         if (sblock->no_io_error_seen)
2433                 scrub_recheck_block_checksum(sblock);
2434
2435         if (!sblock->no_io_error_seen) {
2436                 spin_lock(&sctx->stat_lock);
2437                 sctx->stat.read_errors++;
2438                 spin_unlock(&sctx->stat_lock);
2439                 btrfs_err_rl_in_rcu(fs_info,
2440                         "IO error rebuilding logical %llu for dev %s",
2441                         logical, rcu_str_deref(dev->name));
2442         } else if (sblock->header_error || sblock->checksum_error) {
2443                 spin_lock(&sctx->stat_lock);
2444                 sctx->stat.uncorrectable_errors++;
2445                 spin_unlock(&sctx->stat_lock);
2446                 btrfs_err_rl_in_rcu(fs_info,
2447                         "failed to rebuild valid logical %llu for dev %s",
2448                         logical, rcu_str_deref(dev->name));
2449         } else {
2450                 scrub_write_block_to_dev_replace(sblock);
2451         }
2452
2453         scrub_block_put(sblock);
2454
2455         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2456                 mutex_lock(&sctx->wr_lock);
2457                 scrub_wr_submit(sctx);
2458                 mutex_unlock(&sctx->wr_lock);
2459         }
2460
2461         scrub_pending_bio_dec(sctx);
2462 }
2463
2464 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2465 {
2466         struct scrub_ctx *sctx = sblock->sctx;
2467         struct btrfs_fs_info *fs_info = sctx->fs_info;
2468         u64 length = sblock->page_count * PAGE_SIZE;
2469         u64 logical = sblock->pagev[0]->logical;
2470         struct btrfs_bio *bbio = NULL;
2471         struct bio *bio;
2472         struct btrfs_raid_bio *rbio;
2473         int ret;
2474         int i;
2475
2476         btrfs_bio_counter_inc_blocked(fs_info);
2477         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
2478                         &length, &bbio);
2479         if (ret || !bbio || !bbio->raid_map)
2480                 goto bbio_out;
2481
2482         if (WARN_ON(!sctx->is_dev_replace ||
2483                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2484                 /*
2485                  * We shouldn't be scrubbing a missing device. Even for dev
2486                  * replace, we should only get here for RAID 5/6. We either
2487                  * managed to mount something with no mirrors remaining or
2488                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2489                  */
2490                 goto bbio_out;
2491         }
2492
2493         bio = btrfs_io_bio_alloc(0);
2494         bio->bi_iter.bi_sector = logical >> 9;
2495         bio->bi_private = sblock;
2496         bio->bi_end_io = scrub_missing_raid56_end_io;
2497
2498         rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length);
2499         if (!rbio)
2500                 goto rbio_out;
2501
2502         for (i = 0; i < sblock->page_count; i++) {
2503                 struct scrub_page *spage = sblock->pagev[i];
2504
2505                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2506         }
2507
2508         btrfs_init_work(&sblock->work, btrfs_scrub_helper,
2509                         scrub_missing_raid56_worker, NULL, NULL);
2510         scrub_block_get(sblock);
2511         scrub_pending_bio_inc(sctx);
2512         raid56_submit_missing_rbio(rbio);
2513         return;
2514
2515 rbio_out:
2516         bio_put(bio);
2517 bbio_out:
2518         btrfs_bio_counter_dec(fs_info);
2519         btrfs_put_bbio(bbio);
2520         spin_lock(&sctx->stat_lock);
2521         sctx->stat.malloc_errors++;
2522         spin_unlock(&sctx->stat_lock);
2523 }
2524
2525 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2526                        u64 physical, struct btrfs_device *dev, u64 flags,
2527                        u64 gen, int mirror_num, u8 *csum, int force,
2528                        u64 physical_for_dev_replace)
2529 {
2530         struct scrub_block *sblock;
2531         int index;
2532
2533         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2534         if (!sblock) {
2535                 spin_lock(&sctx->stat_lock);
2536                 sctx->stat.malloc_errors++;
2537                 spin_unlock(&sctx->stat_lock);
2538                 return -ENOMEM;
2539         }
2540
2541         /* one ref inside this function, plus one for each page added to
2542          * a bio later on */
2543         refcount_set(&sblock->refs, 1);
2544         sblock->sctx = sctx;
2545         sblock->no_io_error_seen = 1;
2546
2547         for (index = 0; len > 0; index++) {
2548                 struct scrub_page *spage;
2549                 u64 l = min_t(u64, len, PAGE_SIZE);
2550
2551                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2552                 if (!spage) {
2553 leave_nomem:
2554                         spin_lock(&sctx->stat_lock);
2555                         sctx->stat.malloc_errors++;
2556                         spin_unlock(&sctx->stat_lock);
2557                         scrub_block_put(sblock);
2558                         return -ENOMEM;
2559                 }
2560                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2561                 scrub_page_get(spage);
2562                 sblock->pagev[index] = spage;
2563                 spage->sblock = sblock;
2564                 spage->dev = dev;
2565                 spage->flags = flags;
2566                 spage->generation = gen;
2567                 spage->logical = logical;
2568                 spage->physical = physical;
2569                 spage->physical_for_dev_replace = physical_for_dev_replace;
2570                 spage->mirror_num = mirror_num;
2571                 if (csum) {
2572                         spage->have_csum = 1;
2573                         memcpy(spage->csum, csum, sctx->csum_size);
2574                 } else {
2575                         spage->have_csum = 0;
2576                 }
2577                 sblock->page_count++;
2578                 spage->page = alloc_page(GFP_KERNEL);
2579                 if (!spage->page)
2580                         goto leave_nomem;
2581                 len -= l;
2582                 logical += l;
2583                 physical += l;
2584                 physical_for_dev_replace += l;
2585         }
2586
2587         WARN_ON(sblock->page_count == 0);
2588         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2589                 /*
2590                  * This case should only be hit for RAID 5/6 device replace. See
2591                  * the comment in scrub_missing_raid56_pages() for details.
2592                  */
2593                 scrub_missing_raid56_pages(sblock);
2594         } else {
2595                 for (index = 0; index < sblock->page_count; index++) {
2596                         struct scrub_page *spage = sblock->pagev[index];
2597                         int ret;
2598
2599                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2600                         if (ret) {
2601                                 scrub_block_put(sblock);
2602                                 return ret;
2603                         }
2604                 }
2605
2606                 if (force)
2607                         scrub_submit(sctx);
2608         }
2609
2610         /* last one frees, either here or in bio completion for last page */
2611         scrub_block_put(sblock);
2612         return 0;
2613 }
2614
2615 static void scrub_bio_end_io(struct bio *bio)
2616 {
2617         struct scrub_bio *sbio = bio->bi_private;
2618         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2619
2620         sbio->status = bio->bi_status;
2621         sbio->bio = bio;
2622
2623         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2624 }
2625
2626 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2627 {
2628         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2629         struct scrub_ctx *sctx = sbio->sctx;
2630         int i;
2631
2632         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2633         if (sbio->status) {
2634                 for (i = 0; i < sbio->page_count; i++) {
2635                         struct scrub_page *spage = sbio->pagev[i];
2636
2637                         spage->io_error = 1;
2638                         spage->sblock->no_io_error_seen = 0;
2639                 }
2640         }
2641
2642         /* now complete the scrub_block items that have all pages completed */
2643         for (i = 0; i < sbio->page_count; i++) {
2644                 struct scrub_page *spage = sbio->pagev[i];
2645                 struct scrub_block *sblock = spage->sblock;
2646
2647                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2648                         scrub_block_complete(sblock);
2649                 scrub_block_put(sblock);
2650         }
2651
2652         bio_put(sbio->bio);
2653         sbio->bio = NULL;
2654         spin_lock(&sctx->list_lock);
2655         sbio->next_free = sctx->first_free;
2656         sctx->first_free = sbio->index;
2657         spin_unlock(&sctx->list_lock);
2658
2659         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2660                 mutex_lock(&sctx->wr_lock);
2661                 scrub_wr_submit(sctx);
2662                 mutex_unlock(&sctx->wr_lock);
2663         }
2664
2665         scrub_pending_bio_dec(sctx);
2666 }
2667
2668 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2669                                        unsigned long *bitmap,
2670                                        u64 start, u64 len)
2671 {
2672         u64 offset;
2673         u64 nsectors64;
2674         u32 nsectors;
2675         int sectorsize = sparity->sctx->fs_info->sectorsize;
2676
2677         if (len >= sparity->stripe_len) {
2678                 bitmap_set(bitmap, 0, sparity->nsectors);
2679                 return;
2680         }
2681
2682         start -= sparity->logic_start;
2683         start = div64_u64_rem(start, sparity->stripe_len, &offset);
2684         offset = div_u64(offset, sectorsize);
2685         nsectors64 = div_u64(len, sectorsize);
2686
2687         ASSERT(nsectors64 < UINT_MAX);
2688         nsectors = (u32)nsectors64;
2689
2690         if (offset + nsectors <= sparity->nsectors) {
2691                 bitmap_set(bitmap, offset, nsectors);
2692                 return;
2693         }
2694
2695         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2696         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2697 }
2698
2699 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2700                                                    u64 start, u64 len)
2701 {
2702         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2703 }
2704
2705 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2706                                                   u64 start, u64 len)
2707 {
2708         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2709 }
2710
2711 static void scrub_block_complete(struct scrub_block *sblock)
2712 {
2713         int corrupted = 0;
2714
2715         if (!sblock->no_io_error_seen) {
2716                 corrupted = 1;
2717                 scrub_handle_errored_block(sblock);
2718         } else {
2719                 /*
2720                  * if has checksum error, write via repair mechanism in
2721                  * dev replace case, otherwise write here in dev replace
2722                  * case.
2723                  */
2724                 corrupted = scrub_checksum(sblock);
2725                 if (!corrupted && sblock->sctx->is_dev_replace)
2726                         scrub_write_block_to_dev_replace(sblock);
2727         }
2728
2729         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2730                 u64 start = sblock->pagev[0]->logical;
2731                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2732                           PAGE_SIZE;
2733
2734                 scrub_parity_mark_sectors_error(sblock->sparity,
2735                                                 start, end - start);
2736         }
2737 }
2738
2739 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
2740 {
2741         struct btrfs_ordered_sum *sum = NULL;
2742         unsigned long index;
2743         unsigned long num_sectors;
2744
2745         while (!list_empty(&sctx->csum_list)) {
2746                 sum = list_first_entry(&sctx->csum_list,
2747                                        struct btrfs_ordered_sum, list);
2748                 if (sum->bytenr > logical)
2749                         return 0;
2750                 if (sum->bytenr + sum->len > logical)
2751                         break;
2752
2753                 ++sctx->stat.csum_discards;
2754                 list_del(&sum->list);
2755                 kfree(sum);
2756                 sum = NULL;
2757         }
2758         if (!sum)
2759                 return 0;
2760
2761         index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize);
2762         ASSERT(index < UINT_MAX);
2763
2764         num_sectors = sum->len / sctx->fs_info->sectorsize;
2765         memcpy(csum, sum->sums + index, sctx->csum_size);
2766         if (index == num_sectors - 1) {
2767                 list_del(&sum->list);
2768                 kfree(sum);
2769         }
2770         return 1;
2771 }
2772
2773 /* scrub extent tries to collect up to 64 kB for each bio */
2774 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
2775                         u64 logical, u64 len,
2776                         u64 physical, struct btrfs_device *dev, u64 flags,
2777                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2778 {
2779         int ret;
2780         u8 csum[BTRFS_CSUM_SIZE];
2781         u32 blocksize;
2782
2783         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2784                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2785                         blocksize = map->stripe_len;
2786                 else
2787                         blocksize = sctx->fs_info->sectorsize;
2788                 spin_lock(&sctx->stat_lock);
2789                 sctx->stat.data_extents_scrubbed++;
2790                 sctx->stat.data_bytes_scrubbed += len;
2791                 spin_unlock(&sctx->stat_lock);
2792         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2793                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2794                         blocksize = map->stripe_len;
2795                 else
2796                         blocksize = sctx->fs_info->nodesize;
2797                 spin_lock(&sctx->stat_lock);
2798                 sctx->stat.tree_extents_scrubbed++;
2799                 sctx->stat.tree_bytes_scrubbed += len;
2800                 spin_unlock(&sctx->stat_lock);
2801         } else {
2802                 blocksize = sctx->fs_info->sectorsize;
2803                 WARN_ON(1);
2804         }
2805
2806         while (len) {
2807                 u64 l = min_t(u64, len, blocksize);
2808                 int have_csum = 0;
2809
2810                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2811                         /* push csums to sbio */
2812                         have_csum = scrub_find_csum(sctx, logical, csum);
2813                         if (have_csum == 0)
2814                                 ++sctx->stat.no_csum;
2815                         if (sctx->is_dev_replace && !have_csum) {
2816                                 ret = copy_nocow_pages(sctx, logical, l,
2817                                                        mirror_num,
2818                                                       physical_for_dev_replace);
2819                                 goto behind_scrub_pages;
2820                         }
2821                 }
2822                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2823                                   mirror_num, have_csum ? csum : NULL, 0,
2824                                   physical_for_dev_replace);
2825 behind_scrub_pages:
2826                 if (ret)
2827                         return ret;
2828                 len -= l;
2829                 logical += l;
2830                 physical += l;
2831                 physical_for_dev_replace += l;
2832         }
2833         return 0;
2834 }
2835
2836 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2837                                   u64 logical, u64 len,
2838                                   u64 physical, struct btrfs_device *dev,
2839                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2840 {
2841         struct scrub_ctx *sctx = sparity->sctx;
2842         struct scrub_block *sblock;
2843         int index;
2844
2845         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2846         if (!sblock) {
2847                 spin_lock(&sctx->stat_lock);
2848                 sctx->stat.malloc_errors++;
2849                 spin_unlock(&sctx->stat_lock);
2850                 return -ENOMEM;
2851         }
2852
2853         /* one ref inside this function, plus one for each page added to
2854          * a bio later on */
2855         refcount_set(&sblock->refs, 1);
2856         sblock->sctx = sctx;
2857         sblock->no_io_error_seen = 1;
2858         sblock->sparity = sparity;
2859         scrub_parity_get(sparity);
2860
2861         for (index = 0; len > 0; index++) {
2862                 struct scrub_page *spage;
2863                 u64 l = min_t(u64, len, PAGE_SIZE);
2864
2865                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2866                 if (!spage) {
2867 leave_nomem:
2868                         spin_lock(&sctx->stat_lock);
2869                         sctx->stat.malloc_errors++;
2870                         spin_unlock(&sctx->stat_lock);
2871                         scrub_block_put(sblock);
2872                         return -ENOMEM;
2873                 }
2874                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2875                 /* For scrub block */
2876                 scrub_page_get(spage);
2877                 sblock->pagev[index] = spage;
2878                 /* For scrub parity */
2879                 scrub_page_get(spage);
2880                 list_add_tail(&spage->list, &sparity->spages);
2881                 spage->sblock = sblock;
2882                 spage->dev = dev;
2883                 spage->flags = flags;
2884                 spage->generation = gen;
2885                 spage->logical = logical;
2886                 spage->physical = physical;
2887                 spage->mirror_num = mirror_num;
2888                 if (csum) {
2889                         spage->have_csum = 1;
2890                         memcpy(spage->csum, csum, sctx->csum_size);
2891                 } else {
2892                         spage->have_csum = 0;
2893                 }
2894                 sblock->page_count++;
2895                 spage->page = alloc_page(GFP_KERNEL);
2896                 if (!spage->page)
2897                         goto leave_nomem;
2898                 len -= l;
2899                 logical += l;
2900                 physical += l;
2901         }
2902
2903         WARN_ON(sblock->page_count == 0);
2904         for (index = 0; index < sblock->page_count; index++) {
2905                 struct scrub_page *spage = sblock->pagev[index];
2906                 int ret;
2907
2908                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2909                 if (ret) {
2910                         scrub_block_put(sblock);
2911                         return ret;
2912                 }
2913         }
2914
2915         /* last one frees, either here or in bio completion for last page */
2916         scrub_block_put(sblock);
2917         return 0;
2918 }
2919
2920 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2921                                    u64 logical, u64 len,
2922                                    u64 physical, struct btrfs_device *dev,
2923                                    u64 flags, u64 gen, int mirror_num)
2924 {
2925         struct scrub_ctx *sctx = sparity->sctx;
2926         int ret;
2927         u8 csum[BTRFS_CSUM_SIZE];
2928         u32 blocksize;
2929
2930         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2931                 scrub_parity_mark_sectors_error(sparity, logical, len);
2932                 return 0;
2933         }
2934
2935         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2936                 blocksize = sparity->stripe_len;
2937         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2938                 blocksize = sparity->stripe_len;
2939         } else {
2940                 blocksize = sctx->fs_info->sectorsize;
2941                 WARN_ON(1);
2942         }
2943
2944         while (len) {
2945                 u64 l = min_t(u64, len, blocksize);
2946                 int have_csum = 0;
2947
2948                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2949                         /* push csums to sbio */
2950                         have_csum = scrub_find_csum(sctx, logical, csum);
2951                         if (have_csum == 0)
2952                                 goto skip;
2953                 }
2954                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2955                                              flags, gen, mirror_num,
2956                                              have_csum ? csum : NULL);
2957                 if (ret)
2958                         return ret;
2959 skip:
2960                 len -= l;
2961                 logical += l;
2962                 physical += l;
2963         }
2964         return 0;
2965 }
2966
2967 /*
2968  * Given a physical address, this will calculate it's
2969  * logical offset. if this is a parity stripe, it will return
2970  * the most left data stripe's logical offset.
2971  *
2972  * return 0 if it is a data stripe, 1 means parity stripe.
2973  */
2974 static int get_raid56_logic_offset(u64 physical, int num,
2975                                    struct map_lookup *map, u64 *offset,
2976                                    u64 *stripe_start)
2977 {
2978         int i;
2979         int j = 0;
2980         u64 stripe_nr;
2981         u64 last_offset;
2982         u32 stripe_index;
2983         u32 rot;
2984
2985         last_offset = (physical - map->stripes[num].physical) *
2986                       nr_data_stripes(map);
2987         if (stripe_start)
2988                 *stripe_start = last_offset;
2989
2990         *offset = last_offset;
2991         for (i = 0; i < nr_data_stripes(map); i++) {
2992                 *offset = last_offset + i * map->stripe_len;
2993
2994                 stripe_nr = div64_u64(*offset, map->stripe_len);
2995                 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2996
2997                 /* Work out the disk rotation on this stripe-set */
2998                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2999                 /* calculate which stripe this data locates */
3000                 rot += i;
3001                 stripe_index = rot % map->num_stripes;
3002                 if (stripe_index == num)
3003                         return 0;
3004                 if (stripe_index < num)
3005                         j++;
3006         }
3007         *offset = last_offset + j * map->stripe_len;
3008         return 1;
3009 }
3010
3011 static void scrub_free_parity(struct scrub_parity *sparity)
3012 {
3013         struct scrub_ctx *sctx = sparity->sctx;
3014         struct scrub_page *curr, *next;
3015         int nbits;
3016
3017         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
3018         if (nbits) {
3019                 spin_lock(&sctx->stat_lock);
3020                 sctx->stat.read_errors += nbits;
3021                 sctx->stat.uncorrectable_errors += nbits;
3022                 spin_unlock(&sctx->stat_lock);
3023         }
3024
3025         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
3026                 list_del_init(&curr->list);
3027                 scrub_page_put(curr);
3028         }
3029
3030         kfree(sparity);
3031 }
3032
3033 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
3034 {
3035         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
3036                                                     work);
3037         struct scrub_ctx *sctx = sparity->sctx;
3038
3039         scrub_free_parity(sparity);
3040         scrub_pending_bio_dec(sctx);
3041 }
3042
3043 static void scrub_parity_bio_endio(struct bio *bio)
3044 {
3045         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
3046         struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
3047
3048         if (bio->bi_status)
3049                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
3050                           sparity->nsectors);
3051
3052         bio_put(bio);
3053
3054         btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
3055                         scrub_parity_bio_endio_worker, NULL, NULL);
3056         btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
3057 }
3058
3059 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
3060 {
3061         struct scrub_ctx *sctx = sparity->sctx;
3062         struct btrfs_fs_info *fs_info = sctx->fs_info;
3063         struct bio *bio;
3064         struct btrfs_raid_bio *rbio;
3065         struct btrfs_bio *bbio = NULL;
3066         u64 length;
3067         int ret;
3068
3069         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
3070                            sparity->nsectors))
3071                 goto out;
3072
3073         length = sparity->logic_end - sparity->logic_start;
3074
3075         btrfs_bio_counter_inc_blocked(fs_info);
3076         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
3077                                &length, &bbio);
3078         if (ret || !bbio || !bbio->raid_map)
3079                 goto bbio_out;
3080
3081         bio = btrfs_io_bio_alloc(0);
3082         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
3083         bio->bi_private = sparity;
3084         bio->bi_end_io = scrub_parity_bio_endio;
3085
3086         rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio,
3087                                               length, sparity->scrub_dev,
3088                                               sparity->dbitmap,
3089                                               sparity->nsectors);
3090         if (!rbio)
3091                 goto rbio_out;
3092
3093         scrub_pending_bio_inc(sctx);
3094         raid56_parity_submit_scrub_rbio(rbio);
3095         return;
3096
3097 rbio_out:
3098         bio_put(bio);
3099 bbio_out:
3100         btrfs_bio_counter_dec(fs_info);
3101         btrfs_put_bbio(bbio);
3102         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
3103                   sparity->nsectors);
3104         spin_lock(&sctx->stat_lock);
3105         sctx->stat.malloc_errors++;
3106         spin_unlock(&sctx->stat_lock);
3107 out:
3108         scrub_free_parity(sparity);
3109 }
3110
3111 static inline int scrub_calc_parity_bitmap_len(int nsectors)
3112 {
3113         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
3114 }
3115
3116 static void scrub_parity_get(struct scrub_parity *sparity)
3117 {
3118         refcount_inc(&sparity->refs);
3119 }
3120
3121 static void scrub_parity_put(struct scrub_parity *sparity)
3122 {
3123         if (!refcount_dec_and_test(&sparity->refs))
3124                 return;
3125
3126         scrub_parity_check_and_repair(sparity);
3127 }
3128
3129 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
3130                                                   struct map_lookup *map,
3131                                                   struct btrfs_device *sdev,
3132                                                   struct btrfs_path *path,
3133                                                   u64 logic_start,
3134                                                   u64 logic_end)
3135 {
3136         struct btrfs_fs_info *fs_info = sctx->fs_info;
3137         struct btrfs_root *root = fs_info->extent_root;
3138         struct btrfs_root *csum_root = fs_info->csum_root;
3139         struct btrfs_extent_item *extent;
3140         struct btrfs_bio *bbio = NULL;
3141         u64 flags;
3142         int ret;
3143         int slot;
3144         struct extent_buffer *l;
3145         struct btrfs_key key;
3146         u64 generation;
3147         u64 extent_logical;
3148         u64 extent_physical;
3149         u64 extent_len;
3150         u64 mapped_length;
3151         struct btrfs_device *extent_dev;
3152         struct scrub_parity *sparity;
3153         int nsectors;
3154         int bitmap_len;
3155         int extent_mirror_num;
3156         int stop_loop = 0;
3157
3158         nsectors = div_u64(map->stripe_len, fs_info->sectorsize);
3159         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
3160         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
3161                           GFP_NOFS);
3162         if (!sparity) {
3163                 spin_lock(&sctx->stat_lock);
3164                 sctx->stat.malloc_errors++;
3165                 spin_unlock(&sctx->stat_lock);
3166                 return -ENOMEM;
3167         }
3168
3169         sparity->stripe_len = map->stripe_len;
3170         sparity->nsectors = nsectors;
3171         sparity->sctx = sctx;
3172         sparity->scrub_dev = sdev;
3173         sparity->logic_start = logic_start;
3174         sparity->logic_end = logic_end;
3175         refcount_set(&sparity->refs, 1);
3176         INIT_LIST_HEAD(&sparity->spages);
3177         sparity->dbitmap = sparity->bitmap;
3178         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
3179
3180         ret = 0;
3181         while (logic_start < logic_end) {
3182                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3183                         key.type = BTRFS_METADATA_ITEM_KEY;
3184                 else
3185                         key.type = BTRFS_EXTENT_ITEM_KEY;
3186                 key.objectid = logic_start;
3187                 key.offset = (u64)-1;
3188
3189                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3190                 if (ret < 0)
3191                         goto out;
3192
3193                 if (ret > 0) {
3194                         ret = btrfs_previous_extent_item(root, path, 0);
3195                         if (ret < 0)
3196                                 goto out;
3197                         if (ret > 0) {
3198                                 btrfs_release_path(path);
3199                                 ret = btrfs_search_slot(NULL, root, &key,
3200                                                         path, 0, 0);
3201                                 if (ret < 0)
3202                                         goto out;
3203                         }
3204                 }
3205
3206                 stop_loop = 0;
3207                 while (1) {
3208                         u64 bytes;
3209
3210                         l = path->nodes[0];
3211                         slot = path->slots[0];
3212                         if (slot >= btrfs_header_nritems(l)) {
3213                                 ret = btrfs_next_leaf(root, path);
3214                                 if (ret == 0)
3215                                         continue;
3216                                 if (ret < 0)
3217                                         goto out;
3218
3219                                 stop_loop = 1;
3220                                 break;
3221                         }
3222                         btrfs_item_key_to_cpu(l, &key, slot);
3223
3224                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3225                             key.type != BTRFS_METADATA_ITEM_KEY)
3226                                 goto next;
3227
3228                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3229                                 bytes = fs_info->nodesize;
3230                         else
3231                                 bytes = key.offset;
3232
3233                         if (key.objectid + bytes <= logic_start)
3234                                 goto next;
3235
3236                         if (key.objectid >= logic_end) {
3237                                 stop_loop = 1;
3238                                 break;
3239                         }
3240
3241                         while (key.objectid >= logic_start + map->stripe_len)
3242                                 logic_start += map->stripe_len;
3243
3244                         extent = btrfs_item_ptr(l, slot,
3245                                                 struct btrfs_extent_item);
3246                         flags = btrfs_extent_flags(l, extent);
3247                         generation = btrfs_extent_generation(l, extent);
3248
3249                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3250                             (key.objectid < logic_start ||
3251                              key.objectid + bytes >
3252                              logic_start + map->stripe_len)) {
3253                                 btrfs_err(fs_info,
3254                                           "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3255                                           key.objectid, logic_start);
3256                                 spin_lock(&sctx->stat_lock);
3257                                 sctx->stat.uncorrectable_errors++;
3258                                 spin_unlock(&sctx->stat_lock);
3259                                 goto next;
3260                         }
3261 again:
3262                         extent_logical = key.objectid;
3263                         extent_len = bytes;
3264
3265                         if (extent_logical < logic_start) {
3266                                 extent_len -= logic_start - extent_logical;
3267                                 extent_logical = logic_start;
3268                         }
3269
3270                         if (extent_logical + extent_len >
3271                             logic_start + map->stripe_len)
3272                                 extent_len = logic_start + map->stripe_len -
3273                                              extent_logical;
3274
3275                         scrub_parity_mark_sectors_data(sparity, extent_logical,
3276                                                        extent_len);
3277
3278                         mapped_length = extent_len;
3279                         bbio = NULL;
3280                         ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
3281                                         extent_logical, &mapped_length, &bbio,
3282                                         0);
3283                         if (!ret) {
3284                                 if (!bbio || mapped_length < extent_len)
3285                                         ret = -EIO;
3286                         }
3287                         if (ret) {
3288                                 btrfs_put_bbio(bbio);
3289                                 goto out;
3290                         }
3291                         extent_physical = bbio->stripes[0].physical;
3292                         extent_mirror_num = bbio->mirror_num;
3293                         extent_dev = bbio->stripes[0].dev;
3294                         btrfs_put_bbio(bbio);
3295
3296                         ret = btrfs_lookup_csums_range(csum_root,
3297                                                 extent_logical,
3298                                                 extent_logical + extent_len - 1,
3299                                                 &sctx->csum_list, 1);
3300                         if (ret)
3301                                 goto out;
3302
3303                         ret = scrub_extent_for_parity(sparity, extent_logical,
3304                                                       extent_len,
3305                                                       extent_physical,
3306                                                       extent_dev, flags,
3307                                                       generation,
3308                                                       extent_mirror_num);
3309
3310                         scrub_free_csums(sctx);
3311
3312                         if (ret)
3313                                 goto out;
3314
3315                         if (extent_logical + extent_len <
3316                             key.objectid + bytes) {
3317                                 logic_start += map->stripe_len;
3318
3319                                 if (logic_start >= logic_end) {
3320                                         stop_loop = 1;
3321                                         break;
3322                                 }
3323
3324                                 if (logic_start < key.objectid + bytes) {
3325                                         cond_resched();
3326                                         goto again;
3327                                 }
3328                         }
3329 next:
3330                         path->slots[0]++;
3331                 }
3332
3333                 btrfs_release_path(path);
3334
3335                 if (stop_loop)
3336                         break;
3337
3338                 logic_start += map->stripe_len;
3339         }
3340 out:
3341         if (ret < 0)
3342                 scrub_parity_mark_sectors_error(sparity, logic_start,
3343                                                 logic_end - logic_start);
3344         scrub_parity_put(sparity);
3345         scrub_submit(sctx);
3346         mutex_lock(&sctx->wr_lock);
3347         scrub_wr_submit(sctx);
3348         mutex_unlock(&sctx->wr_lock);
3349
3350         btrfs_release_path(path);
3351         return ret < 0 ? ret : 0;
3352 }
3353
3354 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3355                                            struct map_lookup *map,
3356                                            struct btrfs_device *scrub_dev,
3357                                            int num, u64 base, u64 length,
3358                                            int is_dev_replace)
3359 {
3360         struct btrfs_path *path, *ppath;
3361         struct btrfs_fs_info *fs_info = sctx->fs_info;
3362         struct btrfs_root *root = fs_info->extent_root;
3363         struct btrfs_root *csum_root = fs_info->csum_root;
3364         struct btrfs_extent_item *extent;
3365         struct blk_plug plug;
3366         u64 flags;
3367         int ret;
3368         int slot;
3369         u64 nstripes;
3370         struct extent_buffer *l;
3371         u64 physical;
3372         u64 logical;
3373         u64 logic_end;
3374         u64 physical_end;
3375         u64 generation;
3376         int mirror_num;
3377         struct reada_control *reada1;
3378         struct reada_control *reada2;
3379         struct btrfs_key key;
3380         struct btrfs_key key_end;
3381         u64 increment = map->stripe_len;
3382         u64 offset;
3383         u64 extent_logical;
3384         u64 extent_physical;
3385         u64 extent_len;
3386         u64 stripe_logical;
3387         u64 stripe_end;
3388         struct btrfs_device *extent_dev;
3389         int extent_mirror_num;
3390         int stop_loop = 0;
3391
3392         physical = map->stripes[num].physical;
3393         offset = 0;
3394         nstripes = div64_u64(length, map->stripe_len);
3395         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3396                 offset = map->stripe_len * num;
3397                 increment = map->stripe_len * map->num_stripes;
3398                 mirror_num = 1;
3399         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3400                 int factor = map->num_stripes / map->sub_stripes;
3401                 offset = map->stripe_len * (num / map->sub_stripes);
3402                 increment = map->stripe_len * factor;
3403                 mirror_num = num % map->sub_stripes + 1;
3404         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3405                 increment = map->stripe_len;
3406                 mirror_num = num % map->num_stripes + 1;
3407         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3408                 increment = map->stripe_len;
3409                 mirror_num = num % map->num_stripes + 1;
3410         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3411                 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3412                 increment = map->stripe_len * nr_data_stripes(map);
3413                 mirror_num = 1;
3414         } else {
3415                 increment = map->stripe_len;
3416                 mirror_num = 1;
3417         }
3418
3419         path = btrfs_alloc_path();
3420         if (!path)
3421                 return -ENOMEM;
3422
3423         ppath = btrfs_alloc_path();
3424         if (!ppath) {
3425                 btrfs_free_path(path);
3426                 return -ENOMEM;
3427         }
3428
3429         /*
3430          * work on commit root. The related disk blocks are static as
3431          * long as COW is applied. This means, it is save to rewrite
3432          * them to repair disk errors without any race conditions
3433          */
3434         path->search_commit_root = 1;
3435         path->skip_locking = 1;
3436
3437         ppath->search_commit_root = 1;
3438         ppath->skip_locking = 1;
3439         /*
3440          * trigger the readahead for extent tree csum tree and wait for
3441          * completion. During readahead, the scrub is officially paused
3442          * to not hold off transaction commits
3443          */
3444         logical = base + offset;
3445         physical_end = physical + nstripes * map->stripe_len;
3446         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3447                 get_raid56_logic_offset(physical_end, num,
3448                                         map, &logic_end, NULL);
3449                 logic_end += base;
3450         } else {
3451                 logic_end = logical + increment * nstripes;
3452         }
3453         wait_event(sctx->list_wait,
3454                    atomic_read(&sctx->bios_in_flight) == 0);
3455         scrub_blocked_if_needed(fs_info);
3456
3457         /* FIXME it might be better to start readahead at commit root */
3458         key.objectid = logical;
3459         key.type = BTRFS_EXTENT_ITEM_KEY;
3460         key.offset = (u64)0;
3461         key_end.objectid = logic_end;
3462         key_end.type = BTRFS_METADATA_ITEM_KEY;
3463         key_end.offset = (u64)-1;
3464         reada1 = btrfs_reada_add(root, &key, &key_end);
3465
3466         key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3467         key.type = BTRFS_EXTENT_CSUM_KEY;
3468         key.offset = logical;
3469         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3470         key_end.type = BTRFS_EXTENT_CSUM_KEY;
3471         key_end.offset = logic_end;
3472         reada2 = btrfs_reada_add(csum_root, &key, &key_end);
3473
3474         if (!IS_ERR(reada1))
3475                 btrfs_reada_wait(reada1);
3476         if (!IS_ERR(reada2))
3477                 btrfs_reada_wait(reada2);
3478
3479
3480         /*
3481          * collect all data csums for the stripe to avoid seeking during
3482          * the scrub. This might currently (crc32) end up to be about 1MB
3483          */
3484         blk_start_plug(&plug);
3485
3486         /*
3487          * now find all extents for each stripe and scrub them
3488          */
3489         ret = 0;
3490         while (physical < physical_end) {
3491                 /*
3492                  * canceled?
3493                  */
3494                 if (atomic_read(&fs_info->scrub_cancel_req) ||
3495                     atomic_read(&sctx->cancel_req)) {
3496                         ret = -ECANCELED;
3497                         goto out;
3498                 }
3499                 /*
3500                  * check to see if we have to pause
3501                  */
3502                 if (atomic_read(&fs_info->scrub_pause_req)) {
3503                         /* push queued extents */
3504                         sctx->flush_all_writes = true;
3505                         scrub_submit(sctx);
3506                         mutex_lock(&sctx->wr_lock);
3507                         scrub_wr_submit(sctx);
3508                         mutex_unlock(&sctx->wr_lock);
3509                         wait_event(sctx->list_wait,
3510                                    atomic_read(&sctx->bios_in_flight) == 0);
3511                         sctx->flush_all_writes = false;
3512                         scrub_blocked_if_needed(fs_info);
3513                 }
3514
3515                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3516                         ret = get_raid56_logic_offset(physical, num, map,
3517                                                       &logical,
3518                                                       &stripe_logical);
3519                         logical += base;
3520                         if (ret) {
3521                                 /* it is parity strip */
3522                                 stripe_logical += base;
3523                                 stripe_end = stripe_logical + increment;
3524                                 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3525                                                           ppath, stripe_logical,
3526                                                           stripe_end);
3527                                 if (ret)
3528                                         goto out;
3529                                 goto skip;
3530                         }
3531                 }
3532
3533                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3534                         key.type = BTRFS_METADATA_ITEM_KEY;
3535                 else
3536                         key.type = BTRFS_EXTENT_ITEM_KEY;
3537                 key.objectid = logical;
3538                 key.offset = (u64)-1;
3539
3540                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3541                 if (ret < 0)
3542                         goto out;
3543
3544                 if (ret > 0) {
3545                         ret = btrfs_previous_extent_item(root, path, 0);
3546                         if (ret < 0)
3547                                 goto out;
3548                         if (ret > 0) {
3549                                 /* there's no smaller item, so stick with the
3550                                  * larger one */
3551                                 btrfs_release_path(path);
3552                                 ret = btrfs_search_slot(NULL, root, &key,
3553                                                         path, 0, 0);
3554                                 if (ret < 0)
3555                                         goto out;
3556                         }
3557                 }
3558
3559                 stop_loop = 0;
3560                 while (1) {
3561                         u64 bytes;
3562
3563                         l = path->nodes[0];
3564                         slot = path->slots[0];
3565                         if (slot >= btrfs_header_nritems(l)) {
3566                                 ret = btrfs_next_leaf(root, path);
3567                                 if (ret == 0)
3568                                         continue;
3569                                 if (ret < 0)
3570                                         goto out;
3571
3572                                 stop_loop = 1;
3573                                 break;
3574                         }
3575                         btrfs_item_key_to_cpu(l, &key, slot);
3576
3577                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3578                             key.type != BTRFS_METADATA_ITEM_KEY)
3579                                 goto next;
3580
3581                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3582                                 bytes = fs_info->nodesize;
3583                         else
3584                                 bytes = key.offset;
3585
3586                         if (key.objectid + bytes <= logical)
3587                                 goto next;
3588
3589                         if (key.objectid >= logical + map->stripe_len) {
3590                                 /* out of this device extent */
3591                                 if (key.objectid >= logic_end)
3592                                         stop_loop = 1;
3593                                 break;
3594                         }
3595
3596                         extent = btrfs_item_ptr(l, slot,
3597                                                 struct btrfs_extent_item);
3598                         flags = btrfs_extent_flags(l, extent);
3599                         generation = btrfs_extent_generation(l, extent);
3600
3601                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3602                             (key.objectid < logical ||
3603                              key.objectid + bytes >
3604                              logical + map->stripe_len)) {
3605                                 btrfs_err(fs_info,
3606                                            "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3607                                        key.objectid, logical);
3608                                 spin_lock(&sctx->stat_lock);
3609                                 sctx->stat.uncorrectable_errors++;
3610                                 spin_unlock(&sctx->stat_lock);
3611                                 goto next;
3612                         }
3613
3614 again:
3615                         extent_logical = key.objectid;
3616                         extent_len = bytes;
3617
3618                         /*
3619                          * trim extent to this stripe
3620                          */
3621                         if (extent_logical < logical) {
3622                                 extent_len -= logical - extent_logical;
3623                                 extent_logical = logical;
3624                         }
3625                         if (extent_logical + extent_len >
3626                             logical + map->stripe_len) {
3627                                 extent_len = logical + map->stripe_len -
3628                                              extent_logical;
3629                         }
3630
3631                         extent_physical = extent_logical - logical + physical;
3632                         extent_dev = scrub_dev;
3633                         extent_mirror_num = mirror_num;
3634                         if (is_dev_replace)
3635                                 scrub_remap_extent(fs_info, extent_logical,
3636                                                    extent_len, &extent_physical,
3637                                                    &extent_dev,
3638                                                    &extent_mirror_num);
3639
3640                         ret = btrfs_lookup_csums_range(csum_root,
3641                                                        extent_logical,
3642                                                        extent_logical +
3643                                                        extent_len - 1,
3644                                                        &sctx->csum_list, 1);
3645                         if (ret)
3646                                 goto out;
3647
3648                         ret = scrub_extent(sctx, map, extent_logical, extent_len,
3649                                            extent_physical, extent_dev, flags,
3650                                            generation, extent_mirror_num,
3651                                            extent_logical - logical + physical);
3652
3653                         scrub_free_csums(sctx);
3654
3655                         if (ret)
3656                                 goto out;
3657
3658                         if (extent_logical + extent_len <
3659                             key.objectid + bytes) {
3660                                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3661                                         /*
3662                                          * loop until we find next data stripe
3663                                          * or we have finished all stripes.
3664                                          */
3665 loop:
3666                                         physical += map->stripe_len;
3667                                         ret = get_raid56_logic_offset(physical,
3668                                                         num, map, &logical,
3669                                                         &stripe_logical);
3670                                         logical += base;
3671
3672                                         if (ret && physical < physical_end) {
3673                                                 stripe_logical += base;
3674                                                 stripe_end = stripe_logical +
3675                                                                 increment;
3676                                                 ret = scrub_raid56_parity(sctx,
3677                                                         map, scrub_dev, ppath,
3678                                                         stripe_logical,
3679                                                         stripe_end);
3680                                                 if (ret)
3681                                                         goto out;
3682                                                 goto loop;
3683                                         }
3684                                 } else {
3685                                         physical += map->stripe_len;
3686                                         logical += increment;
3687                                 }
3688                                 if (logical < key.objectid + bytes) {
3689                                         cond_resched();
3690                                         goto again;
3691                                 }
3692
3693                                 if (physical >= physical_end) {
3694                                         stop_loop = 1;
3695                                         break;
3696                                 }
3697                         }
3698 next:
3699                         path->slots[0]++;
3700                 }
3701                 btrfs_release_path(path);
3702 skip:
3703                 logical += increment;
3704                 physical += map->stripe_len;
3705                 spin_lock(&sctx->stat_lock);
3706                 if (stop_loop)
3707                         sctx->stat.last_physical = map->stripes[num].physical +
3708                                                    length;
3709                 else
3710                         sctx->stat.last_physical = physical;
3711                 spin_unlock(&sctx->stat_lock);
3712                 if (stop_loop)
3713                         break;
3714         }
3715 out:
3716         /* push queued extents */
3717         scrub_submit(sctx);
3718         mutex_lock(&sctx->wr_lock);
3719         scrub_wr_submit(sctx);
3720         mutex_unlock(&sctx->wr_lock);
3721
3722         blk_finish_plug(&plug);
3723         btrfs_free_path(path);
3724         btrfs_free_path(ppath);
3725         return ret < 0 ? ret : 0;
3726 }
3727
3728 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3729                                           struct btrfs_device *scrub_dev,
3730                                           u64 chunk_offset, u64 length,
3731                                           u64 dev_offset,
3732                                           struct btrfs_block_group_cache *cache,
3733                                           int is_dev_replace)
3734 {
3735         struct btrfs_fs_info *fs_info = sctx->fs_info;
3736         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
3737         struct map_lookup *map;
3738         struct extent_map *em;
3739         int i;
3740         int ret = 0;
3741
3742         read_lock(&map_tree->map_tree.lock);
3743         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3744         read_unlock(&map_tree->map_tree.lock);
3745
3746         if (!em) {
3747                 /*
3748                  * Might have been an unused block group deleted by the cleaner
3749                  * kthread or relocation.
3750                  */
3751                 spin_lock(&cache->lock);
3752                 if (!cache->removed)
3753                         ret = -EINVAL;
3754                 spin_unlock(&cache->lock);
3755
3756                 return ret;
3757         }
3758
3759         map = em->map_lookup;
3760         if (em->start != chunk_offset)
3761                 goto out;
3762
3763         if (em->len < length)
3764                 goto out;
3765
3766         for (i = 0; i < map->num_stripes; ++i) {
3767                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3768                     map->stripes[i].physical == dev_offset) {
3769                         ret = scrub_stripe(sctx, map, scrub_dev, i,
3770                                            chunk_offset, length,
3771                                            is_dev_replace);
3772                         if (ret)
3773                                 goto out;
3774                 }
3775         }
3776 out:
3777         free_extent_map(em);
3778
3779         return ret;
3780 }
3781
3782 static noinline_for_stack
3783 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3784                            struct btrfs_device *scrub_dev, u64 start, u64 end,
3785                            int is_dev_replace)
3786 {
3787         struct btrfs_dev_extent *dev_extent = NULL;
3788         struct btrfs_path *path;
3789         struct btrfs_fs_info *fs_info = sctx->fs_info;
3790         struct btrfs_root *root = fs_info->dev_root;
3791         u64 length;
3792         u64 chunk_offset;
3793         int ret = 0;
3794         int ro_set;
3795         int slot;
3796         struct extent_buffer *l;
3797         struct btrfs_key key;
3798         struct btrfs_key found_key;
3799         struct btrfs_block_group_cache *cache;
3800         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
3801
3802         path = btrfs_alloc_path();
3803         if (!path)
3804                 return -ENOMEM;
3805
3806         path->reada = READA_FORWARD;
3807         path->search_commit_root = 1;
3808         path->skip_locking = 1;
3809
3810         key.objectid = scrub_dev->devid;
3811         key.offset = 0ull;
3812         key.type = BTRFS_DEV_EXTENT_KEY;
3813
3814         while (1) {
3815                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3816                 if (ret < 0)
3817                         break;
3818                 if (ret > 0) {
3819                         if (path->slots[0] >=
3820                             btrfs_header_nritems(path->nodes[0])) {
3821                                 ret = btrfs_next_leaf(root, path);
3822                                 if (ret < 0)
3823                                         break;
3824                                 if (ret > 0) {
3825                                         ret = 0;
3826                                         break;
3827                                 }
3828                         } else {
3829                                 ret = 0;
3830                         }
3831                 }
3832
3833                 l = path->nodes[0];
3834                 slot = path->slots[0];
3835
3836                 btrfs_item_key_to_cpu(l, &found_key, slot);
3837
3838                 if (found_key.objectid != scrub_dev->devid)
3839                         break;
3840
3841                 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
3842                         break;
3843
3844                 if (found_key.offset >= end)
3845                         break;
3846
3847                 if (found_key.offset < key.offset)
3848                         break;
3849
3850                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3851                 length = btrfs_dev_extent_length(l, dev_extent);
3852
3853                 if (found_key.offset + length <= start)
3854                         goto skip;
3855
3856                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3857
3858                 /*
3859                  * get a reference on the corresponding block group to prevent
3860                  * the chunk from going away while we scrub it
3861                  */
3862                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3863
3864                 /* some chunks are removed but not committed to disk yet,
3865                  * continue scrubbing */
3866                 if (!cache)
3867                         goto skip;
3868
3869                 /*
3870                  * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3871                  * to avoid deadlock caused by:
3872                  * btrfs_inc_block_group_ro()
3873                  * -> btrfs_wait_for_commit()
3874                  * -> btrfs_commit_transaction()
3875                  * -> btrfs_scrub_pause()
3876                  */
3877                 scrub_pause_on(fs_info);
3878                 ret = btrfs_inc_block_group_ro(fs_info, cache);
3879                 if (!ret && is_dev_replace) {
3880                         /*
3881                          * If we are doing a device replace wait for any tasks
3882                          * that started dellaloc right before we set the block
3883                          * group to RO mode, as they might have just allocated
3884                          * an extent from it or decided they could do a nocow
3885                          * write. And if any such tasks did that, wait for their
3886                          * ordered extents to complete and then commit the
3887                          * current transaction, so that we can later see the new
3888                          * extent items in the extent tree - the ordered extents
3889                          * create delayed data references (for cow writes) when
3890                          * they complete, which will be run and insert the
3891                          * corresponding extent items into the extent tree when
3892                          * we commit the transaction they used when running
3893                          * inode.c:btrfs_finish_ordered_io(). We later use
3894                          * the commit root of the extent tree to find extents
3895                          * to copy from the srcdev into the tgtdev, and we don't
3896                          * want to miss any new extents.
3897                          */
3898                         btrfs_wait_block_group_reservations(cache);
3899                         btrfs_wait_nocow_writers(cache);
3900                         ret = btrfs_wait_ordered_roots(fs_info, U64_MAX,
3901                                                        cache->key.objectid,
3902                                                        cache->key.offset);
3903                         if (ret > 0) {
3904                                 struct btrfs_trans_handle *trans;
3905
3906                                 trans = btrfs_join_transaction(root);
3907                                 if (IS_ERR(trans))
3908                                         ret = PTR_ERR(trans);
3909                                 else
3910                                         ret = btrfs_commit_transaction(trans);
3911                                 if (ret) {
3912                                         scrub_pause_off(fs_info);
3913                                         btrfs_put_block_group(cache);
3914                                         break;
3915                                 }
3916                         }
3917                 }
3918                 scrub_pause_off(fs_info);
3919
3920                 if (ret == 0) {
3921                         ro_set = 1;
3922                 } else if (ret == -ENOSPC) {
3923                         /*
3924                          * btrfs_inc_block_group_ro return -ENOSPC when it
3925                          * failed in creating new chunk for metadata.
3926                          * It is not a problem for scrub/replace, because
3927                          * metadata are always cowed, and our scrub paused
3928                          * commit_transactions.
3929                          */
3930                         ro_set = 0;
3931                 } else {
3932                         btrfs_warn(fs_info,
3933                                    "failed setting block group ro: %d", ret);
3934                         btrfs_put_block_group(cache);
3935                         break;
3936                 }
3937
3938                 btrfs_dev_replace_write_lock(&fs_info->dev_replace);
3939                 dev_replace->cursor_right = found_key.offset + length;
3940                 dev_replace->cursor_left = found_key.offset;
3941                 dev_replace->item_needs_writeback = 1;
3942                 btrfs_dev_replace_write_unlock(&fs_info->dev_replace);
3943                 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
3944                                   found_key.offset, cache, is_dev_replace);
3945
3946                 /*
3947                  * flush, submit all pending read and write bios, afterwards
3948                  * wait for them.
3949                  * Note that in the dev replace case, a read request causes
3950                  * write requests that are submitted in the read completion
3951                  * worker. Therefore in the current situation, it is required
3952                  * that all write requests are flushed, so that all read and
3953                  * write requests are really completed when bios_in_flight
3954                  * changes to 0.
3955                  */
3956                 sctx->flush_all_writes = true;
3957                 scrub_submit(sctx);
3958                 mutex_lock(&sctx->wr_lock);
3959                 scrub_wr_submit(sctx);
3960                 mutex_unlock(&sctx->wr_lock);
3961
3962                 wait_event(sctx->list_wait,
3963                            atomic_read(&sctx->bios_in_flight) == 0);
3964
3965                 scrub_pause_on(fs_info);
3966
3967                 /*
3968                  * must be called before we decrease @scrub_paused.
3969                  * make sure we don't block transaction commit while
3970                  * we are waiting pending workers finished.
3971                  */
3972                 wait_event(sctx->list_wait,
3973                            atomic_read(&sctx->workers_pending) == 0);
3974                 sctx->flush_all_writes = false;
3975
3976                 scrub_pause_off(fs_info);
3977
3978                 btrfs_dev_replace_write_lock(&fs_info->dev_replace);
3979                 dev_replace->cursor_left = dev_replace->cursor_right;
3980                 dev_replace->item_needs_writeback = 1;
3981                 btrfs_dev_replace_write_unlock(&fs_info->dev_replace);
3982
3983                 if (ro_set)
3984                         btrfs_dec_block_group_ro(cache);
3985
3986                 /*
3987                  * We might have prevented the cleaner kthread from deleting
3988                  * this block group if it was already unused because we raced
3989                  * and set it to RO mode first. So add it back to the unused
3990                  * list, otherwise it might not ever be deleted unless a manual
3991                  * balance is triggered or it becomes used and unused again.
3992                  */
3993                 spin_lock(&cache->lock);
3994                 if (!cache->removed && !cache->ro && cache->reserved == 0 &&
3995                     btrfs_block_group_used(&cache->item) == 0) {
3996                         spin_unlock(&cache->lock);
3997                         spin_lock(&fs_info->unused_bgs_lock);
3998                         if (list_empty(&cache->bg_list)) {
3999                                 btrfs_get_block_group(cache);
4000                                 list_add_tail(&cache->bg_list,
4001                                               &fs_info->unused_bgs);
4002                         }
4003                         spin_unlock(&fs_info->unused_bgs_lock);
4004                 } else {
4005                         spin_unlock(&cache->lock);
4006                 }
4007
4008                 btrfs_put_block_group(cache);
4009                 if (ret)
4010                         break;
4011                 if (is_dev_replace &&
4012                     atomic64_read(&dev_replace->num_write_errors) > 0) {
4013                         ret = -EIO;
4014                         break;
4015                 }
4016                 if (sctx->stat.malloc_errors > 0) {
4017                         ret = -ENOMEM;
4018                         break;
4019                 }
4020 skip:
4021                 key.offset = found_key.offset + length;
4022                 btrfs_release_path(path);
4023         }
4024
4025         btrfs_free_path(path);
4026
4027         return ret;
4028 }
4029
4030 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
4031                                            struct btrfs_device *scrub_dev)
4032 {
4033         int     i;
4034         u64     bytenr;
4035         u64     gen;
4036         int     ret;
4037         struct btrfs_fs_info *fs_info = sctx->fs_info;
4038
4039         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
4040                 return -EIO;
4041
4042         /* Seed devices of a new filesystem has their own generation. */
4043         if (scrub_dev->fs_devices != fs_info->fs_devices)
4044                 gen = scrub_dev->generation;
4045         else
4046                 gen = fs_info->last_trans_committed;
4047
4048         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
4049                 bytenr = btrfs_sb_offset(i);
4050                 if (bytenr + BTRFS_SUPER_INFO_SIZE >
4051                     scrub_dev->commit_total_bytes)
4052                         break;
4053
4054                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
4055                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
4056                                   NULL, 1, bytenr);
4057                 if (ret)
4058                         return ret;
4059         }
4060         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
4061
4062         return 0;
4063 }
4064
4065 /*
4066  * get a reference count on fs_info->scrub_workers. start worker if necessary
4067  */
4068 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
4069                                                 int is_dev_replace)
4070 {
4071         unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
4072         int max_active = fs_info->thread_pool_size;
4073
4074         if (fs_info->scrub_workers_refcnt == 0) {
4075                 fs_info->scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub",
4076                                 flags, is_dev_replace ? 1 : max_active, 4);
4077                 if (!fs_info->scrub_workers)
4078                         goto fail_scrub_workers;
4079
4080                 fs_info->scrub_wr_completion_workers =
4081                         btrfs_alloc_workqueue(fs_info, "scrubwrc", flags,
4082                                               max_active, 2);
4083                 if (!fs_info->scrub_wr_completion_workers)
4084                         goto fail_scrub_wr_completion_workers;
4085
4086                 fs_info->scrub_nocow_workers =
4087                         btrfs_alloc_workqueue(fs_info, "scrubnc", flags, 1, 0);
4088                 if (!fs_info->scrub_nocow_workers)
4089                         goto fail_scrub_nocow_workers;
4090                 fs_info->scrub_parity_workers =
4091                         btrfs_alloc_workqueue(fs_info, "scrubparity", flags,
4092                                               max_active, 2);
4093                 if (!fs_info->scrub_parity_workers)
4094                         goto fail_scrub_parity_workers;
4095         }
4096         ++fs_info->scrub_workers_refcnt;
4097         return 0;
4098
4099 fail_scrub_parity_workers:
4100         btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
4101 fail_scrub_nocow_workers:
4102         btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
4103 fail_scrub_wr_completion_workers:
4104         btrfs_destroy_workqueue(fs_info->scrub_workers);
4105 fail_scrub_workers:
4106         return -ENOMEM;
4107 }
4108
4109 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
4110 {
4111         if (--fs_info->scrub_workers_refcnt == 0) {
4112                 btrfs_destroy_workqueue(fs_info->scrub_workers);
4113                 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
4114                 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
4115                 btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
4116         }
4117         WARN_ON(fs_info->scrub_workers_refcnt < 0);
4118 }
4119
4120 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
4121                     u64 end, struct btrfs_scrub_progress *progress,
4122                     int readonly, int is_dev_replace)
4123 {
4124         struct scrub_ctx *sctx;
4125         int ret;
4126         struct btrfs_device *dev;
4127         struct rcu_string *name;
4128
4129         if (btrfs_fs_closing(fs_info))
4130                 return -EINVAL;
4131
4132         if (fs_info->nodesize > BTRFS_STRIPE_LEN) {
4133                 /*
4134                  * in this case scrub is unable to calculate the checksum
4135                  * the way scrub is implemented. Do not handle this
4136                  * situation at all because it won't ever happen.
4137                  */
4138                 btrfs_err(fs_info,
4139                            "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
4140                        fs_info->nodesize,
4141                        BTRFS_STRIPE_LEN);
4142                 return -EINVAL;
4143         }
4144
4145         if (fs_info->sectorsize != PAGE_SIZE) {
4146                 /* not supported for data w/o checksums */
4147                 btrfs_err_rl(fs_info,
4148                            "scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails",
4149                        fs_info->sectorsize, PAGE_SIZE);
4150                 return -EINVAL;
4151         }
4152
4153         if (fs_info->nodesize >
4154             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
4155             fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
4156                 /*
4157                  * would exhaust the array bounds of pagev member in
4158                  * struct scrub_block
4159                  */
4160                 btrfs_err(fs_info,
4161                           "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
4162                        fs_info->nodesize,
4163                        SCRUB_MAX_PAGES_PER_BLOCK,
4164                        fs_info->sectorsize,
4165                        SCRUB_MAX_PAGES_PER_BLOCK);
4166                 return -EINVAL;
4167         }
4168
4169
4170         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4171         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
4172         if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
4173                      !is_dev_replace)) {
4174                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4175                 return -ENODEV;
4176         }
4177
4178         if (!is_dev_replace && !readonly &&
4179             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
4180                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4181                 rcu_read_lock();
4182                 name = rcu_dereference(dev->name);
4183                 btrfs_err(fs_info, "scrub: device %s is not writable",
4184                           name->str);
4185                 rcu_read_unlock();
4186                 return -EROFS;
4187         }
4188
4189         mutex_lock(&fs_info->scrub_lock);
4190         if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4191             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {
4192                 mutex_unlock(&fs_info->scrub_lock);
4193                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4194                 return -EIO;
4195         }
4196
4197         btrfs_dev_replace_read_lock(&fs_info->dev_replace);
4198         if (dev->scrub_ctx ||
4199             (!is_dev_replace &&
4200              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
4201                 btrfs_dev_replace_read_unlock(&fs_info->dev_replace);
4202                 mutex_unlock(&fs_info->scrub_lock);
4203                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4204                 return -EINPROGRESS;
4205         }
4206         btrfs_dev_replace_read_unlock(&fs_info->dev_replace);
4207
4208         ret = scrub_workers_get(fs_info, is_dev_replace);
4209         if (ret) {
4210                 mutex_unlock(&fs_info->scrub_lock);
4211                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4212                 return ret;
4213         }
4214
4215         sctx = scrub_setup_ctx(dev, is_dev_replace);
4216         if (IS_ERR(sctx)) {
4217                 mutex_unlock(&fs_info->scrub_lock);
4218                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4219                 scrub_workers_put(fs_info);
4220                 return PTR_ERR(sctx);
4221         }
4222         sctx->readonly = readonly;
4223         dev->scrub_ctx = sctx;
4224         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4225
4226         /*
4227          * checking @scrub_pause_req here, we can avoid
4228          * race between committing transaction and scrubbing.
4229          */
4230         __scrub_blocked_if_needed(fs_info);
4231         atomic_inc(&fs_info->scrubs_running);
4232         mutex_unlock(&fs_info->scrub_lock);
4233
4234         if (!is_dev_replace) {
4235                 /*
4236                  * by holding device list mutex, we can
4237                  * kick off writing super in log tree sync.
4238                  */
4239                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
4240                 ret = scrub_supers(sctx, dev);
4241                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4242         }
4243
4244         if (!ret)
4245                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
4246                                              is_dev_replace);
4247
4248         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
4249         atomic_dec(&fs_info->scrubs_running);
4250         wake_up(&fs_info->scrub_pause_wait);
4251
4252         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
4253
4254         if (progress)
4255                 memcpy(progress, &sctx->stat, sizeof(*progress));
4256
4257         mutex_lock(&fs_info->scrub_lock);
4258         dev->scrub_ctx = NULL;
4259         scrub_workers_put(fs_info);
4260         mutex_unlock(&fs_info->scrub_lock);
4261
4262         scrub_put_ctx(sctx);
4263
4264         return ret;
4265 }
4266
4267 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
4268 {
4269         mutex_lock(&fs_info->scrub_lock);
4270         atomic_inc(&fs_info->scrub_pause_req);
4271         while (atomic_read(&fs_info->scrubs_paused) !=
4272                atomic_read(&fs_info->scrubs_running)) {
4273                 mutex_unlock(&fs_info->scrub_lock);
4274                 wait_event(fs_info->scrub_pause_wait,
4275                            atomic_read(&fs_info->scrubs_paused) ==
4276                            atomic_read(&fs_info->scrubs_running));
4277                 mutex_lock(&fs_info->scrub_lock);
4278         }
4279         mutex_unlock(&fs_info->scrub_lock);
4280 }
4281
4282 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
4283 {
4284         atomic_dec(&fs_info->scrub_pause_req);
4285         wake_up(&fs_info->scrub_pause_wait);
4286 }
4287
4288 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
4289 {
4290         mutex_lock(&fs_info->scrub_lock);
4291         if (!atomic_read(&fs_info->scrubs_running)) {
4292                 mutex_unlock(&fs_info->scrub_lock);
4293                 return -ENOTCONN;
4294         }
4295
4296         atomic_inc(&fs_info->scrub_cancel_req);
4297         while (atomic_read(&fs_info->scrubs_running)) {
4298                 mutex_unlock(&fs_info->scrub_lock);
4299                 wait_event(fs_info->scrub_pause_wait,
4300                            atomic_read(&fs_info->scrubs_running) == 0);
4301                 mutex_lock(&fs_info->scrub_lock);
4302         }
4303         atomic_dec(&fs_info->scrub_cancel_req);
4304         mutex_unlock(&fs_info->scrub_lock);
4305
4306         return 0;
4307 }
4308
4309 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
4310                            struct btrfs_device *dev)
4311 {
4312         struct scrub_ctx *sctx;
4313
4314         mutex_lock(&fs_info->scrub_lock);
4315         sctx = dev->scrub_ctx;
4316         if (!sctx) {
4317                 mutex_unlock(&fs_info->scrub_lock);
4318                 return -ENOTCONN;
4319         }
4320         atomic_inc(&sctx->cancel_req);
4321         while (dev->scrub_ctx) {
4322                 mutex_unlock(&fs_info->scrub_lock);
4323                 wait_event(fs_info->scrub_pause_wait,
4324                            dev->scrub_ctx == NULL);
4325                 mutex_lock(&fs_info->scrub_lock);
4326         }
4327         mutex_unlock(&fs_info->scrub_lock);
4328
4329         return 0;
4330 }
4331
4332 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
4333                          struct btrfs_scrub_progress *progress)
4334 {
4335         struct btrfs_device *dev;
4336         struct scrub_ctx *sctx = NULL;
4337
4338         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4339         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
4340         if (dev)
4341                 sctx = dev->scrub_ctx;
4342         if (sctx)
4343                 memcpy(progress, &sctx->stat, sizeof(*progress));
4344         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4345
4346         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
4347 }
4348
4349 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
4350                                u64 extent_logical, u64 extent_len,
4351                                u64 *extent_physical,
4352                                struct btrfs_device **extent_dev,
4353                                int *extent_mirror_num)
4354 {
4355         u64 mapped_length;
4356         struct btrfs_bio *bbio = NULL;
4357         int ret;
4358
4359         mapped_length = extent_len;
4360         ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,
4361                               &mapped_length, &bbio, 0);
4362         if (ret || !bbio || mapped_length < extent_len ||
4363             !bbio->stripes[0].dev->bdev) {
4364                 btrfs_put_bbio(bbio);
4365                 return;
4366         }
4367
4368         *extent_physical = bbio->stripes[0].physical;
4369         *extent_mirror_num = bbio->mirror_num;
4370         *extent_dev = bbio->stripes[0].dev;
4371         btrfs_put_bbio(bbio);
4372 }
4373
4374 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
4375                             int mirror_num, u64 physical_for_dev_replace)
4376 {
4377         struct scrub_copy_nocow_ctx *nocow_ctx;
4378         struct btrfs_fs_info *fs_info = sctx->fs_info;
4379
4380         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
4381         if (!nocow_ctx) {
4382                 spin_lock(&sctx->stat_lock);
4383                 sctx->stat.malloc_errors++;
4384                 spin_unlock(&sctx->stat_lock);
4385                 return -ENOMEM;
4386         }
4387
4388         scrub_pending_trans_workers_inc(sctx);
4389
4390         nocow_ctx->sctx = sctx;
4391         nocow_ctx->logical = logical;
4392         nocow_ctx->len = len;
4393         nocow_ctx->mirror_num = mirror_num;
4394         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
4395         btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
4396                         copy_nocow_pages_worker, NULL, NULL);
4397         INIT_LIST_HEAD(&nocow_ctx->inodes);
4398         btrfs_queue_work(fs_info->scrub_nocow_workers,
4399                          &nocow_ctx->work);
4400
4401         return 0;
4402 }
4403
4404 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
4405 {
4406         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
4407         struct scrub_nocow_inode *nocow_inode;
4408
4409         nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
4410         if (!nocow_inode)
4411                 return -ENOMEM;
4412         nocow_inode->inum = inum;
4413         nocow_inode->offset = offset;
4414         nocow_inode->root = root;
4415         list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
4416         return 0;
4417 }
4418
4419 #define COPY_COMPLETE 1
4420
4421 static void copy_nocow_pages_worker(struct btrfs_work *work)
4422 {
4423         struct scrub_copy_nocow_ctx *nocow_ctx =
4424                 container_of(work, struct scrub_copy_nocow_ctx, work);
4425         struct scrub_ctx *sctx = nocow_ctx->sctx;
4426         struct btrfs_fs_info *fs_info = sctx->fs_info;
4427         struct btrfs_root *root = fs_info->extent_root;
4428         u64 logical = nocow_ctx->logical;
4429         u64 len = nocow_ctx->len;
4430         int mirror_num = nocow_ctx->mirror_num;
4431         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4432         int ret;
4433         struct btrfs_trans_handle *trans = NULL;
4434         struct btrfs_path *path;
4435         int not_written = 0;
4436
4437         path = btrfs_alloc_path();
4438         if (!path) {
4439                 spin_lock(&sctx->stat_lock);
4440                 sctx->stat.malloc_errors++;
4441                 spin_unlock(&sctx->stat_lock);
4442                 not_written = 1;
4443                 goto out;
4444         }
4445
4446         trans = btrfs_join_transaction(root);
4447         if (IS_ERR(trans)) {
4448                 not_written = 1;
4449                 goto out;
4450         }
4451
4452         ret = iterate_inodes_from_logical(logical, fs_info, path,
4453                         record_inode_for_nocow, nocow_ctx, false);
4454         if (ret != 0 && ret != -ENOENT) {
4455                 btrfs_warn(fs_info,
4456                            "iterate_inodes_from_logical() failed: log %llu, phys %llu, len %llu, mir %u, ret %d",
4457                            logical, physical_for_dev_replace, len, mirror_num,
4458                            ret);
4459                 not_written = 1;
4460                 goto out;
4461         }
4462
4463         btrfs_end_transaction(trans);
4464         trans = NULL;
4465         while (!list_empty(&nocow_ctx->inodes)) {
4466                 struct scrub_nocow_inode *entry;
4467                 entry = list_first_entry(&nocow_ctx->inodes,
4468                                          struct scrub_nocow_inode,
4469                                          list);
4470                 list_del_init(&entry->list);
4471                 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4472                                                  entry->root, nocow_ctx);
4473                 kfree(entry);
4474                 if (ret == COPY_COMPLETE) {
4475                         ret = 0;
4476                         break;
4477                 } else if (ret) {
4478                         break;
4479                 }
4480         }
4481 out:
4482         while (!list_empty(&nocow_ctx->inodes)) {
4483                 struct scrub_nocow_inode *entry;
4484                 entry = list_first_entry(&nocow_ctx->inodes,
4485                                          struct scrub_nocow_inode,
4486                                          list);
4487                 list_del_init(&entry->list);
4488                 kfree(entry);
4489         }
4490         if (trans && !IS_ERR(trans))
4491                 btrfs_end_transaction(trans);
4492         if (not_written)
4493                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4494                                             num_uncorrectable_read_errors);
4495
4496         btrfs_free_path(path);
4497         kfree(nocow_ctx);
4498
4499         scrub_pending_trans_workers_dec(sctx);
4500 }
4501
4502 static int check_extent_to_block(struct btrfs_inode *inode, u64 start, u64 len,
4503                                  u64 logical)
4504 {
4505         struct extent_state *cached_state = NULL;
4506         struct btrfs_ordered_extent *ordered;
4507         struct extent_io_tree *io_tree;
4508         struct extent_map *em;
4509         u64 lockstart = start, lockend = start + len - 1;
4510         int ret = 0;
4511
4512         io_tree = &inode->io_tree;
4513
4514         lock_extent_bits(io_tree, lockstart, lockend, &cached_state);
4515         ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4516         if (ordered) {
4517                 btrfs_put_ordered_extent(ordered);
4518                 ret = 1;
4519                 goto out_unlock;
4520         }
4521
4522         em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4523         if (IS_ERR(em)) {
4524                 ret = PTR_ERR(em);
4525                 goto out_unlock;
4526         }
4527
4528         /*
4529          * This extent does not actually cover the logical extent anymore,
4530          * move on to the next inode.
4531          */
4532         if (em->block_start > logical ||
4533             em->block_start + em->block_len < logical + len ||
4534             test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4535                 free_extent_map(em);
4536                 ret = 1;
4537                 goto out_unlock;
4538         }
4539         free_extent_map(em);
4540
4541 out_unlock:
4542         unlock_extent_cached(io_tree, lockstart, lockend, &cached_state);
4543         return ret;
4544 }
4545
4546 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4547                                       struct scrub_copy_nocow_ctx *nocow_ctx)
4548 {
4549         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->fs_info;
4550         struct btrfs_key key;
4551         struct inode *inode;
4552         struct page *page;
4553         struct btrfs_root *local_root;
4554         struct extent_io_tree *io_tree;
4555         u64 physical_for_dev_replace;
4556         u64 nocow_ctx_logical;
4557         u64 len = nocow_ctx->len;
4558         unsigned long index;
4559         int srcu_index;
4560         int ret = 0;
4561         int err = 0;
4562
4563         key.objectid = root;
4564         key.type = BTRFS_ROOT_ITEM_KEY;
4565         key.offset = (u64)-1;
4566
4567         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4568
4569         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
4570         if (IS_ERR(local_root)) {
4571                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4572                 return PTR_ERR(local_root);
4573         }
4574
4575         key.type = BTRFS_INODE_ITEM_KEY;
4576         key.objectid = inum;
4577         key.offset = 0;
4578         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
4579         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4580         if (IS_ERR(inode))
4581                 return PTR_ERR(inode);
4582
4583         /* Avoid truncate/dio/punch hole.. */
4584         inode_lock(inode);
4585         inode_dio_wait(inode);
4586
4587         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4588         io_tree = &BTRFS_I(inode)->io_tree;
4589         nocow_ctx_logical = nocow_ctx->logical;
4590
4591         ret = check_extent_to_block(BTRFS_I(inode), offset, len,
4592                         nocow_ctx_logical);
4593         if (ret) {
4594                 ret = ret > 0 ? 0 : ret;
4595                 goto out;
4596         }
4597
4598         while (len >= PAGE_SIZE) {
4599                 index = offset >> PAGE_SHIFT;
4600 again:
4601                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4602                 if (!page) {
4603                         btrfs_err(fs_info, "find_or_create_page() failed");
4604                         ret = -ENOMEM;
4605                         goto out;
4606                 }
4607
4608                 if (PageUptodate(page)) {
4609                         if (PageDirty(page))
4610                                 goto next_page;
4611                 } else {
4612                         ClearPageError(page);
4613                         err = extent_read_full_page(io_tree, page,
4614                                                            btrfs_get_extent,
4615                                                            nocow_ctx->mirror_num);
4616                         if (err) {
4617                                 ret = err;
4618                                 goto next_page;
4619                         }
4620
4621                         lock_page(page);
4622                         /*
4623                          * If the page has been remove from the page cache,
4624                          * the data on it is meaningless, because it may be
4625                          * old one, the new data may be written into the new
4626                          * page in the page cache.
4627                          */
4628                         if (page->mapping != inode->i_mapping) {
4629                                 unlock_page(page);
4630                                 put_page(page);
4631                                 goto again;
4632                         }
4633                         if (!PageUptodate(page)) {
4634                                 ret = -EIO;
4635                                 goto next_page;
4636                         }
4637                 }
4638
4639                 ret = check_extent_to_block(BTRFS_I(inode), offset, len,
4640                                             nocow_ctx_logical);
4641                 if (ret) {
4642                         ret = ret > 0 ? 0 : ret;
4643                         goto next_page;
4644                 }
4645
4646                 err = write_page_nocow(nocow_ctx->sctx,
4647                                        physical_for_dev_replace, page);
4648                 if (err)
4649                         ret = err;
4650 next_page:
4651                 unlock_page(page);
4652                 put_page(page);
4653
4654                 if (ret)
4655                         break;
4656
4657                 offset += PAGE_SIZE;
4658                 physical_for_dev_replace += PAGE_SIZE;
4659                 nocow_ctx_logical += PAGE_SIZE;
4660                 len -= PAGE_SIZE;
4661         }
4662         ret = COPY_COMPLETE;
4663 out:
4664         inode_unlock(inode);
4665         iput(inode);
4666         return ret;
4667 }
4668
4669 static int write_page_nocow(struct scrub_ctx *sctx,
4670                             u64 physical_for_dev_replace, struct page *page)
4671 {
4672         struct bio *bio;
4673         struct btrfs_device *dev;
4674
4675         dev = sctx->wr_tgtdev;
4676         if (!dev)
4677                 return -EIO;
4678         if (!dev->bdev) {
4679                 btrfs_warn_rl(dev->fs_info,
4680                         "scrub write_page_nocow(bdev == NULL) is unexpected");
4681                 return -EIO;
4682         }
4683         bio = btrfs_io_bio_alloc(1);
4684         bio->bi_iter.bi_size = 0;
4685         bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
4686         bio_set_dev(bio, dev->bdev);
4687         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
4688         /* bio_add_page won't fail on a freshly allocated bio */
4689         bio_add_page(bio, page, PAGE_SIZE, 0);
4690
4691         if (btrfsic_submit_bio_wait(bio)) {
4692                 bio_put(bio);
4693                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4694                 return -EIO;
4695         }
4696
4697         bio_put(bio);
4698         return 0;
4699 }