fs/btrfs/volumes.c

   1 /*
   2  * Copyright (C) 2007 Oracle.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18 #include <linux/sched.h>
  19 #include <linux/bio.h>
  20 #include <linux/slab.h>
  21 #include <linux/buffer_head.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/random.h>
  24 #include <linux/iocontext.h>
  25 #include <linux/capability.h>
  26 #include <linux/ratelimit.h>
  27 #include <linux/kthread.h>
  28 #include <linux/raid/pq.h>
  29 #include <linux/semaphore.h>
  30 #include <asm/div64.h>
  31 #include "ctree.h"
  32 #include "extent_map.h"
  33 #include "disk-io.h"
  34 #include "transaction.h"
  35 #include "print-tree.h"
  36 #include "volumes.h"
  37 #include "raid56.h"
  38 #include "async-thread.h"
  39 #include "check-integrity.h"
  40 #include "rcu-string.h"
  41 #include "math.h"
  42 #include "dev-replace.h"
  43 #include "sysfs.h"
  44
  45 static int init_first_rw_device(struct btrfs_trans_handle *trans,
  46                                 struct btrfs_root *root,
  47                                 struct btrfs_device *device);
  48 static int btrfs_relocate_sys_chunks(struct btrfs_root *root);
  49 static void __btrfs_reset_dev_stats(struct btrfs_device *dev);
  50 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
  51 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
  52
  53 DEFINE_MUTEX(uuid_mutex);
  54 static LIST_HEAD(fs_uuids);
  55
  56 static struct btrfs_fs_devices *__alloc_fs_devices(void)
  57 {
  58         struct btrfs_fs_devices *fs_devs;
  59
  60         fs_devs = kzalloc(sizeof(*fs_devs), GFP_NOFS);
  61         if (!fs_devs)
  62                 return ERR_PTR(-ENOMEM);
  63
  64         mutex_init(&fs_devs->device_list_mutex);
  65
  66         INIT_LIST_HEAD(&fs_devs->devices);
  67         INIT_LIST_HEAD(&fs_devs->resized_devices);
  68         INIT_LIST_HEAD(&fs_devs->alloc_list);
  69         INIT_LIST_HEAD(&fs_devs->list);
  70
  71         return fs_devs;
  72 }
  73
  74 /**
  75  * alloc_fs_devices - allocate struct btrfs_fs_devices
  76  * @fsid:       a pointer to UUID for this FS.  If NULL a new UUID is
  77  *              generated.
  78  *
  79  * Return: a pointer to a new &struct btrfs_fs_devices on success;
  80  * ERR_PTR() on error.  Returned struct is not linked onto any lists and
  81  * can be destroyed with kfree() right away.
  82  */
  83 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
  84 {
  85         struct btrfs_fs_devices *fs_devs;
  86
  87         fs_devs = __alloc_fs_devices();
  88         if (IS_ERR(fs_devs))
  89                 return fs_devs;
  90
  91         if (fsid)
  92                 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
  93         else
  94                 generate_random_uuid(fs_devs->fsid);
  95
  96         return fs_devs;
  97 }
  98
  99 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
 100 {
 101         struct btrfs_device *device;
 102         WARN_ON(fs_devices->opened);
 103         while (!list_empty(&fs_devices->devices)) {
 104                 device = list_entry(fs_devices->devices.next,
 105                                     struct btrfs_device, dev_list);
 106                 list_del(&device->dev_list);
 107                 rcu_string_free(device->name);
 108                 kfree(device);
 109         }
 110         kfree(fs_devices);
 111 }
 112
 113 static void btrfs_kobject_uevent(struct block_device *bdev,
 114                                  enum kobject_action action)
 115 {
 116         int ret;
 117
 118         ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
 119         if (ret)
 120                 pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
 121                         action,
 122                         kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
 123                         &disk_to_dev(bdev->bd_disk)->kobj);
 124 }
 125
 126 void btrfs_cleanup_fs_uuids(void)
 127 {
 128         struct btrfs_fs_devices *fs_devices;
 129
 130         while (!list_empty(&fs_uuids)) {
 131                 fs_devices = list_entry(fs_uuids.next,
 132                                         struct btrfs_fs_devices, list);
 133                 list_del(&fs_devices->list);
 134                 free_fs_devices(fs_devices);
 135         }
 136 }
 137
 138 static struct btrfs_device *__alloc_device(void)
 139 {
 140         struct btrfs_device *dev;
 141
 142         dev = kzalloc(sizeof(*dev), GFP_NOFS);
 143         if (!dev)
 144                 return ERR_PTR(-ENOMEM);
 145
 146         INIT_LIST_HEAD(&dev->dev_list);
 147         INIT_LIST_HEAD(&dev->dev_alloc_list);
 148         INIT_LIST_HEAD(&dev->resized_list);
 149
 150         spin_lock_init(&dev->io_lock);
 151
 152         spin_lock_init(&dev->reada_lock);
 153         atomic_set(&dev->reada_in_flight, 0);
 154         atomic_set(&dev->dev_stats_ccnt, 0);
 155         INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_WAIT);
 156         INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_WAIT);
 157
 158         return dev;
 159 }
 160
 161 static noinline struct btrfs_device *__find_device(struct list_head *head,
 162                                                    u64 devid, u8 *uuid)
 163 {
 164         struct btrfs_device *dev;
 165
 166         list_for_each_entry(dev, head, dev_list) {
 167                 if (dev->devid == devid &&
 168                     (!uuid || !memcmp(dev->uuid, uuid, BTRFS_UUID_SIZE))) {
 169                         return dev;
 170                 }
 171         }
 172         return NULL;
 173 }
 174
 175 static noinline struct btrfs_fs_devices *find_fsid(u8 *fsid)
 176 {
 177         struct btrfs_fs_devices *fs_devices;
 178
 179         list_for_each_entry(fs_devices, &fs_uuids, list) {
 180                 if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
 181                         return fs_devices;
 182         }
 183         return NULL;
 184 }
 185
 186 static int
 187 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
 188                       int flush, struct block_device **bdev,
 189                       struct buffer_head **bh)
 190 {
 191         int ret;
 192
 193         *bdev = blkdev_get_by_path(device_path, flags, holder);
 194
 195         if (IS_ERR(*bdev)) {
 196                 ret = PTR_ERR(*bdev);
 197                 printk(KERN_INFO "BTRFS: open %s failed\n", device_path);
 198                 goto error;
 199         }
 200
 201         if (flush)
 202                 filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
 203         ret = set_blocksize(*bdev, 4096);
 204         if (ret) {
 205                 blkdev_put(*bdev, flags);
 206                 goto error;
 207         }
 208         invalidate_bdev(*bdev);
 209         *bh = btrfs_read_dev_super(*bdev);
 210         if (!*bh) {
 211                 ret = -EINVAL;
 212                 blkdev_put(*bdev, flags);
 213                 goto error;
 214         }
 215
 216         return 0;
 217
 218 error:
 219         *bdev = NULL;
 220         *bh = NULL;
 221         return ret;
 222 }
 223
 224 static void requeue_list(struct btrfs_pending_bios *pending_bios,
 225                         struct bio *head, struct bio *tail)
 226 {
 227
 228         struct bio *old_head;
 229
 230         old_head = pending_bios->head;
 231         pending_bios->head = head;
 232         if (pending_bios->tail)
 233                 tail->bi_next = old_head;
 234         else
 235                 pending_bios->tail = tail;
 236 }
 237
 238 /*
 239  * we try to collect pending bios for a device so we don't get a large
 240  * number of procs sending bios down to the same device.  This greatly
 241  * improves the schedulers ability to collect and merge the bios.
 242  *
 243  * But, it also turns into a long list of bios to process and that is sure
 244  * to eventually make the worker thread block.  The solution here is to
 245  * make some progress and then put this work struct back at the end of
 246  * the list if the block device is congested.  This way, multiple devices
 247  * can make progress from a single worker thread.
 248  */
 249 static noinline void run_scheduled_bios(struct btrfs_device *device)
 250 {
 251         struct bio *pending;
 252         struct backing_dev_info *bdi;
 253         struct btrfs_fs_info *fs_info;
 254         struct btrfs_pending_bios *pending_bios;
 255         struct bio *tail;
 256         struct bio *cur;
 257         int again = 0;
 258         unsigned long num_run;
 259         unsigned long batch_run = 0;
 260         unsigned long limit;
 261         unsigned long last_waited = 0;
 262         int force_reg = 0;
 263         int sync_pending = 0;
 264         struct blk_plug plug;
 265
 266         /*
 267          * this function runs all the bios we've collected for
 268          * a particular device.  We don't want to wander off to
 269          * another device without first sending all of these down.
 270          * So, setup a plug here and finish it off before we return
 271          */
 272         blk_start_plug(&plug);
 273
 274         bdi = blk_get_backing_dev_info(device->bdev);
 275         fs_info = device->dev_root->fs_info;
 276         limit = btrfs_async_submit_limit(fs_info);
 277         limit = limit * 2 / 3;
 278
 279 loop:
 280         spin_lock(&device->io_lock);
 281
 282 loop_lock:
 283         num_run = 0;
 284
 285         /* take all the bios off the list at once and process them
 286          * later on (without the lock held).  But, remember the
 287          * tail and other pointers so the bios can be properly reinserted
 288          * into the list if we hit congestion
 289          */
 290         if (!force_reg && device->pending_sync_bios.head) {
 291                 pending_bios = &device->pending_sync_bios;
 292                 force_reg = 1;
 293         } else {
 294                 pending_bios = &device->pending_bios;
 295                 force_reg = 0;
 296         }
 297
 298         pending = pending_bios->head;
 299         tail = pending_bios->tail;
 300         WARN_ON(pending && !tail);
 301
 302         /*
 303          * if pending was null this time around, no bios need processing
 304          * at all and we can stop.  Otherwise it'll loop back up again
 305          * and do an additional check so no bios are missed.
 306          *
 307          * device->running_pending is used to synchronize with the
 308          * schedule_bio code.
 309          */
 310         if (device->pending_sync_bios.head == NULL &&
 311             device->pending_bios.head == NULL) {
 312                 again = 0;
 313                 device->running_pending = 0;
 314         } else {
 315                 again = 1;
 316                 device->running_pending = 1;
 317         }
 318
 319         pending_bios->head = NULL;
 320         pending_bios->tail = NULL;
 321
 322         spin_unlock(&device->io_lock);
 323
 324         while (pending) {
 325
 326                 rmb();
 327                 /* we want to work on both lists, but do more bios on the
 328                  * sync list than the regular list
 329                  */
 330                 if ((num_run > 32 &&
 331                     pending_bios != &device->pending_sync_bios &&
 332                     device->pending_sync_bios.head) ||
 333                    (num_run > 64 && pending_bios == &device->pending_sync_bios &&
 334                     device->pending_bios.head)) {
 335                         spin_lock(&device->io_lock);
 336                         requeue_list(pending_bios, pending, tail);
 337                         goto loop_lock;
 338                 }
 339
 340                 cur = pending;
 341                 pending = pending->bi_next;
 342                 cur->bi_next = NULL;
 343
 344                 if (atomic_dec_return(&fs_info->nr_async_bios) < limit &&
 345                     waitqueue_active(&fs_info->async_submit_wait))
 346                         wake_up(&fs_info->async_submit_wait);
 347
 348                 BUG_ON(atomic_read(&cur->bi_cnt) == 0);
 349
 350                 /*
 351                  * if we're doing the sync list, record that our
 352                  * plug has some sync requests on it
 353                  *
 354                  * If we're doing the regular list and there are
 355                  * sync requests sitting around, unplug before
 356                  * we add more
 357                  */
 358                 if (pending_bios == &device->pending_sync_bios) {
 359                         sync_pending = 1;
 360                 } else if (sync_pending) {
 361                         blk_finish_plug(&plug);
 362                         blk_start_plug(&plug);
 363                         sync_pending = 0;
 364                 }
 365
 366                 btrfsic_submit_bio(cur->bi_rw, cur);
 367                 num_run++;
 368                 batch_run++;
 369
 370                 cond_resched();
 371
 372                 /*
 373                  * we made progress, there is more work to do and the bdi
 374                  * is now congested.  Back off and let other work structs
 375                  * run instead
 376                  */
 377                 if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
 378                     fs_info->fs_devices->open_devices > 1) {
 379                         struct io_context *ioc;
 380
 381                         ioc = current->io_context;
 382
 383                         /*
 384                          * the main goal here is that we don't want to
 385                          * block if we're going to be able to submit
 386                          * more requests without blocking.
 387                          *
 388                          * This code does two great things, it pokes into
 389                          * the elevator code from a filesystem _and_
 390                          * it makes assumptions about how batching works.
 391                          */
 392                         if (ioc && ioc->nr_batch_requests > 0 &&
 393                             time_before(jiffies, ioc->last_waited + HZ/50UL) &&
 394                             (last_waited == 0 ||
 395                              ioc->last_waited == last_waited)) {
 396                                 /*
 397                                  * we want to go through our batch of
 398                                  * requests and stop.  So, we copy out
 399                                  * the ioc->last_waited time and test
 400                                  * against it before looping
 401                                  */
 402                                 last_waited = ioc->last_waited;
 403                                 cond_resched();
 404                                 continue;
 405                         }
 406                         spin_lock(&device->io_lock);
 407                         requeue_list(pending_bios, pending, tail);
 408                         device->running_pending = 1;
 409
 410                         spin_unlock(&device->io_lock);
 411                         btrfs_queue_work(fs_info->submit_workers,
 412                                          &device->work);
 413                         goto done;
 414                 }
 415                 /* unplug every 64 requests just for good measure */
 416                 if (batch_run % 64 == 0) {
 417                         blk_finish_plug(&plug);
 418                         blk_start_plug(&plug);
 419                         sync_pending = 0;
 420                 }
 421         }
 422
 423         cond_resched();
 424         if (again)
 425                 goto loop;
 426
 427         spin_lock(&device->io_lock);
 428         if (device->pending_bios.head || device->pending_sync_bios.head)
 429                 goto loop_lock;
 430         spin_unlock(&device->io_lock);
 431
 432 done:
 433         blk_finish_plug(&plug);
 434 }
 435
 436 static void pending_bios_fn(struct btrfs_work *work)
 437 {
 438         struct btrfs_device *device;
 439
 440         device = container_of(work, struct btrfs_device, work);
 441         run_scheduled_bios(device);
 442 }
 443
 444 /*
 445  * Add new device to list of registered devices
 446  *
 447  * Returns:
 448  * 1   - first time device is seen
 449  * 0   - device already known
 450  * < 0 - error
 451  */
 452 static noinline int device_list_add(const char *path,
 453                            struct btrfs_super_block *disk_super,
 454                            u64 devid, struct btrfs_fs_devices **fs_devices_ret)
 455 {
 456         struct btrfs_device *device;
 457         struct btrfs_fs_devices *fs_devices;
 458         struct rcu_string *name;
 459         int ret = 0;
 460         u64 found_transid = btrfs_super_generation(disk_super);
 461
 462         fs_devices = find_fsid(disk_super->fsid);
 463         if (!fs_devices) {
 464                 fs_devices = alloc_fs_devices(disk_super->fsid);
 465                 if (IS_ERR(fs_devices))
 466                         return PTR_ERR(fs_devices);
 467
 468                 list_add(&fs_devices->list, &fs_uuids);
 469
 470                 device = NULL;
 471         } else {
 472                 device = __find_device(&fs_devices->devices, devid,
 473                                        disk_super->dev_item.uuid);
 474         }
 475
 476         if (!device) {
 477                 if (fs_devices->opened)
 478                         return -EBUSY;
 479
 480                 device = btrfs_alloc_device(NULL, &devid,
 481                                             disk_super->dev_item.uuid);
 482                 if (IS_ERR(device)) {
 483                         /* we can safely leave the fs_devices entry around */
 484                         return PTR_ERR(device);
 485                 }
 486
 487                 name = rcu_string_strdup(path, GFP_NOFS);
 488                 if (!name) {
 489                         kfree(device);
 490                         return -ENOMEM;
 491                 }
 492                 rcu_assign_pointer(device->name, name);
 493
 494                 mutex_lock(&fs_devices->device_list_mutex);
 495                 list_add_rcu(&device->dev_list, &fs_devices->devices);
 496                 fs_devices->num_devices++;
 497                 mutex_unlock(&fs_devices->device_list_mutex);
 498
 499                 ret = 1;
 500                 device->fs_devices = fs_devices;
 501         } else if (!device->name || strcmp(device->name->str, path)) {
 502                 /*
 503                  * When FS is already mounted.
 504                  * 1. If you are here and if the device->name is NULL that
 505                  *    means this device was missing at time of FS mount.
 506                  * 2. If you are here and if the device->name is different
 507                  *    from 'path' that means either
 508                  *      a. The same device disappeared and reappeared with
 509                  *         different name. or
 510                  *      b. The missing-disk-which-was-replaced, has
 511                  *         reappeared now.
 512                  *
 513                  * We must allow 1 and 2a above. But 2b would be a spurious
 514                  * and unintentional.
 515                  *
 516                  * Further in case of 1 and 2a above, the disk at 'path'
 517                  * would have missed some transaction when it was away and
 518                  * in case of 2a the stale bdev has to be updated as well.
 519                  * 2b must not be allowed at all time.
 520                  */
 521
 522                 /*
 523                  * For now, we do allow update to btrfs_fs_device through the
 524                  * btrfs dev scan cli after FS has been mounted.  We're still
 525                  * tracking a problem where systems fail mount by subvolume id
 526                  * when we reject replacement on a mounted FS.
 527                  */
 528                 if (!fs_devices->opened && found_transid < device->generation) {
 529                         /*
 530                          * That is if the FS is _not_ mounted and if you
 531                          * are here, that means there is more than one
 532                          * disk with same uuid and devid.We keep the one
 533                          * with larger generation number or the last-in if
 534                          * generation are equal.
 535                          */
 536                         return -EEXIST;
 537                 }
 538
 539                 name = rcu_string_strdup(path, GFP_NOFS);
 540                 if (!name)
 541                         return -ENOMEM;
 542                 rcu_string_free(device->name);
 543                 rcu_assign_pointer(device->name, name);
 544                 if (device->missing) {
 545                         fs_devices->missing_devices--;
 546                         device->missing = 0;
 547                 }
 548         }
 549
 550         /*
 551          * Unmount does not free the btrfs_device struct but would zero
 552          * generation along with most of the other members. So just update
 553          * it back. We need it to pick the disk with largest generation
 554          * (as above).
 555          */
 556         if (!fs_devices->opened)
 557                 device->generation = found_transid;
 558
 559         *fs_devices_ret = fs_devices;
 560
 561         return ret;
 562 }
 563
 564 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
 565 {
 566         struct btrfs_fs_devices *fs_devices;
 567         struct btrfs_device *device;
 568         struct btrfs_device *orig_dev;
 569
 570         fs_devices = alloc_fs_devices(orig->fsid);
 571         if (IS_ERR(fs_devices))
 572                 return fs_devices;
 573
 574         mutex_lock(&orig->device_list_mutex);
 575         fs_devices->total_devices = orig->total_devices;
 576
 577         /* We have held the volume lock, it is safe to get the devices. */
 578         list_for_each_entry(orig_dev, &orig->devices, dev_list) {
 579                 struct rcu_string *name;
 580
 581                 device = btrfs_alloc_device(NULL, &orig_dev->devid,
 582                                             orig_dev->uuid);
 583                 if (IS_ERR(device))
 584                         goto error;
 585
 586                 /*
 587                  * This is ok to do without rcu read locked because we hold the
 588                  * uuid mutex so nothing we touch in here is going to disappear.
 589                  */
 590                 if (orig_dev->name) {
 591                         name = rcu_string_strdup(orig_dev->name->str, GFP_NOFS);
 592                         if (!name) {
 593                                 kfree(device);
 594                                 goto error;
 595                         }
 596                         rcu_assign_pointer(device->name, name);
 597                 }
 598
 599                 list_add(&device->dev_list, &fs_devices->devices);
 600                 device->fs_devices = fs_devices;
 601                 fs_devices->num_devices++;
 602         }
 603         mutex_unlock(&orig->device_list_mutex);
 604         return fs_devices;
 605 error:
 606         mutex_unlock(&orig->device_list_mutex);
 607         free_fs_devices(fs_devices);
 608         return ERR_PTR(-ENOMEM);
 609 }
 610
 611 void btrfs_close_extra_devices(struct btrfs_fs_devices *fs_devices, int step)
 612 {
 613         struct btrfs_device *device, *next;
 614         struct btrfs_device *latest_dev = NULL;
 615
 616         mutex_lock(&uuid_mutex);
 617 again:
 618         /* This is the initialized path, it is safe to release the devices. */
 619         list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
 620                 if (device->in_fs_metadata) {
 621                         if (!device->is_tgtdev_for_dev_replace &&
 622                             (!latest_dev ||
 623                              device->generation > latest_dev->generation)) {
 624                                 latest_dev = device;
 625                         }
 626                         continue;
 627                 }
 628
 629                 if (device->devid == BTRFS_DEV_REPLACE_DEVID) {
 630                         /*
 631                          * In the first step, keep the device which has
 632                          * the correct fsid and the devid that is used
 633                          * for the dev_replace procedure.
 634                          * In the second step, the dev_replace state is
 635                          * read from the device tree and it is known
 636                          * whether the procedure is really active or
 637                          * not, which means whether this device is
 638                          * used or whether it should be removed.
 639                          */
 640                         if (step == 0 || device->is_tgtdev_for_dev_replace) {
 641                                 continue;
 642                         }
 643                 }
 644                 if (device->bdev) {
 645                         blkdev_put(device->bdev, device->mode);
 646                         device->bdev = NULL;
 647                         fs_devices->open_devices--;
 648                 }
 649                 if (device->writeable) {
 650                         list_del_init(&device->dev_alloc_list);
 651                         device->writeable = 0;
 652                         if (!device->is_tgtdev_for_dev_replace)
 653                                 fs_devices->rw_devices--;
 654                 }
 655                 list_del_init(&device->dev_list);
 656                 fs_devices->num_devices--;
 657                 rcu_string_free(device->name);
 658                 kfree(device);
 659         }
 660
 661         if (fs_devices->seed) {
 662                 fs_devices = fs_devices->seed;
 663                 goto again;
 664         }
 665
 666         fs_devices->latest_bdev = latest_dev->bdev;
 667
 668         mutex_unlock(&uuid_mutex);
 669 }
 670
 671 static void __free_device(struct work_struct *work)
 672 {
 673         struct btrfs_device *device;
 674
 675         device = container_of(work, struct btrfs_device, rcu_work);
 676
 677         if (device->bdev)
 678                 blkdev_put(device->bdev, device->mode);
 679
 680         rcu_string_free(device->name);
 681         kfree(device);
 682 }
 683
 684 static void free_device(struct rcu_head *head)
 685 {
 686         struct btrfs_device *device;
 687
 688         device = container_of(head, struct btrfs_device, rcu);
 689
 690         INIT_WORK(&device->rcu_work, __free_device);
 691         schedule_work(&device->rcu_work);
 692 }
 693
 694 static int __btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
 695 {
 696         struct btrfs_device *device;
 697
 698         if (--fs_devices->opened > 0)
 699                 return 0;
 700
 701         mutex_lock(&fs_devices->device_list_mutex);
 702         list_for_each_entry(device, &fs_devices->devices, dev_list) {
 703                 struct btrfs_device *new_device;
 704                 struct rcu_string *name;
 705
 706                 if (device->bdev)
 707                         fs_devices->open_devices--;
 708
 709                 if (device->writeable &&
 710                     device->devid != BTRFS_DEV_REPLACE_DEVID) {
 711                         list_del_init(&device->dev_alloc_list);
 712                         fs_devices->rw_devices--;
 713                 }
 714
 715                 if (device->missing)
 716                         fs_devices->missing_devices--;
 717
 718                 new_device = btrfs_alloc_device(NULL, &device->devid,
 719                                                 device->uuid);
 720                 BUG_ON(IS_ERR(new_device)); /* -ENOMEM */
 721
 722                 /* Safe because we are under uuid_mutex */
 723                 if (device->name) {
 724                         name = rcu_string_strdup(device->name->str, GFP_NOFS);
 725                         BUG_ON(!name); /* -ENOMEM */
 726                         rcu_assign_pointer(new_device->name, name);
 727                 }
 728
 729                 list_replace_rcu(&device->dev_list, &new_device->dev_list);
 730                 new_device->fs_devices = device->fs_devices;
 731
 732                 call_rcu(&device->rcu, free_device);
 733         }
 734         mutex_unlock(&fs_devices->device_list_mutex);
 735
 736         WARN_ON(fs_devices->open_devices);
 737         WARN_ON(fs_devices->rw_devices);
 738         fs_devices->opened = 0;
 739         fs_devices->seeding = 0;
 740
 741         return 0;
 742 }
 743
 744 int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
 745 {
 746         struct btrfs_fs_devices *seed_devices = NULL;
 747         int ret;
 748
 749         mutex_lock(&uuid_mutex);
 750         ret = __btrfs_close_devices(fs_devices);
 751         if (!fs_devices->opened) {
 752                 seed_devices = fs_devices->seed;
 753                 fs_devices->seed = NULL;
 754         }
 755         mutex_unlock(&uuid_mutex);
 756
 757         while (seed_devices) {
 758                 fs_devices = seed_devices;
 759                 seed_devices = fs_devices->seed;
 760                 __btrfs_close_devices(fs_devices);
 761                 free_fs_devices(fs_devices);
 762         }
 763         /*
 764          * Wait for rcu kworkers under __btrfs_close_devices
 765          * to finish all blkdev_puts so device is really
 766          * free when umount is done.
 767          */
 768         rcu_barrier();
 769         return ret;
 770 }
 771
 772 static int __btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
 773                                 fmode_t flags, void *holder)
 774 {
 775         struct request_queue *q;
 776         struct block_device *bdev;
 777         struct list_head *head = &fs_devices->devices;
 778         struct btrfs_device *device;
 779         struct btrfs_device *latest_dev = NULL;
 780         struct buffer_head *bh;
 781         struct btrfs_super_block *disk_super;
 782         u64 devid;
 783         int seeding = 1;
 784         int ret = 0;
 785
 786         flags |= FMODE_EXCL;
 787
 788         list_for_each_entry(device, head, dev_list) {
 789                 if (device->bdev)
 790                         continue;
 791                 if (!device->name)
 792                         continue;
 793
 794                 /* Just open everything we can; ignore failures here */
 795                 if (btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
 796                                             &bdev, &bh))
 797                         continue;
 798
 799                 disk_super = (struct btrfs_super_block *)bh->b_data;
 800                 devid = btrfs_stack_device_id(&disk_super->dev_item);
 801                 if (devid != device->devid)
 802                         goto error_brelse;
 803
 804                 if (memcmp(device->uuid, disk_super->dev_item.uuid,
 805                            BTRFS_UUID_SIZE))
 806                         goto error_brelse;
 807
 808                 device->generation = btrfs_super_generation(disk_super);
 809                 if (!latest_dev ||
 810                     device->generation > latest_dev->generation)
 811                         latest_dev = device;
 812
 813                 if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
 814                         device->writeable = 0;
 815                 } else {
 816                         device->writeable = !bdev_read_only(bdev);
 817                         seeding = 0;
 818                 }
 819
 820                 q = bdev_get_queue(bdev);
 821                 if (blk_queue_discard(q))
 822                         device->can_discard = 1;
 823
 824                 device->bdev = bdev;
 825                 device->in_fs_metadata = 0;
 826                 device->mode = flags;
 827
 828                 if (!blk_queue_nonrot(bdev_get_queue(bdev)))
 829                         fs_devices->rotating = 1;
 830
 831                 fs_devices->open_devices++;
 832                 if (device->writeable &&
 833                     device->devid != BTRFS_DEV_REPLACE_DEVID) {
 834                         fs_devices->rw_devices++;
 835                         list_add(&device->dev_alloc_list,
 836                                  &fs_devices->alloc_list);
 837                 }
 838                 brelse(bh);
 839                 continue;
 840
 841 error_brelse:
 842                 brelse(bh);
 843                 blkdev_put(bdev, flags);
 844                 continue;
 845         }
 846         if (fs_devices->open_devices == 0) {
 847                 ret = -EINVAL;
 848                 goto out;
 849         }
 850         fs_devices->seeding = seeding;
 851         fs_devices->opened = 1;
 852         fs_devices->latest_bdev = latest_dev->bdev;
 853         fs_devices->total_rw_bytes = 0;
 854 out:
 855         return ret;
 856 }
 857
 858 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
 859                        fmode_t flags, void *holder)
 860 {
 861         int ret;
 862
 863         mutex_lock(&uuid_mutex);
 864         if (fs_devices->opened) {
 865                 fs_devices->opened++;
 866                 ret = 0;
 867         } else {
 868                 ret = __btrfs_open_devices(fs_devices, flags, holder);
 869         }
 870         mutex_unlock(&uuid_mutex);
 871         return ret;
 872 }
 873
 874 /*
 875  * Look for a btrfs signature on a device. This may be called out of the mount path
 876  * and we are not allowed to call set_blocksize during the scan. The superblock
 877  * is read via pagecache
 878  */
 879 int btrfs_scan_one_device(const char *path, fmode_t flags, void *holder,
 880                           struct btrfs_fs_devices **fs_devices_ret)
 881 {
 882         struct btrfs_super_block *disk_super;
 883         struct block_device *bdev;
 884         struct page *page;
 885         void *p;
 886         int ret = -EINVAL;
 887         u64 devid;
 888         u64 transid;
 889         u64 total_devices;
 890         u64 bytenr;
 891         pgoff_t index;
 892
 893         /*
 894          * we would like to check all the supers, but that would make
 895          * a btrfs mount succeed after a mkfs from a different FS.
 896          * So, we need to add a special mount option to scan for
 897          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
 898          */
 899         bytenr = btrfs_sb_offset(0);
 900         flags |= FMODE_EXCL;
 901         mutex_lock(&uuid_mutex);
 902
 903         bdev = blkdev_get_by_path(path, flags, holder);
 904
 905         if (IS_ERR(bdev)) {
 906                 ret = PTR_ERR(bdev);
 907                 goto error;
 908         }
 909
 910         /* make sure our super fits in the device */
 911         if (bytenr + PAGE_CACHE_SIZE >= i_size_read(bdev->bd_inode))
 912                 goto error_bdev_put;
 913
 914         /* make sure our super fits in the page */
 915         if (sizeof(*disk_super) > PAGE_CACHE_SIZE)
 916                 goto error_bdev_put;
 917
 918         /* make sure our super doesn't straddle pages on disk */
 919         index = bytenr >> PAGE_CACHE_SHIFT;
 920         if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_CACHE_SHIFT != index)
 921                 goto error_bdev_put;
 922
 923         /* pull in the page with our super */
 924         page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
 925                                    index, GFP_NOFS);
 926
 927         if (IS_ERR_OR_NULL(page))
 928                 goto error_bdev_put;
 929
 930         p = kmap(page);
 931
 932         /* align our pointer to the offset of the super block */
 933         disk_super = p + (bytenr & ~PAGE_CACHE_MASK);
 934
 935         if (btrfs_super_bytenr(disk_super) != bytenr ||
 936             btrfs_super_magic(disk_super) != BTRFS_MAGIC)
 937                 goto error_unmap;
 938
 939         devid = btrfs_stack_device_id(&disk_super->dev_item);
 940         transid = btrfs_super_generation(disk_super);
 941         total_devices = btrfs_super_num_devices(disk_super);
 942
 943         ret = device_list_add(path, disk_super, devid, fs_devices_ret);
 944         if (ret > 0) {
 945                 if (disk_super->label[0]) {
 946                         if (disk_super->label[BTRFS_LABEL_SIZE - 1])
 947                                 disk_super->label[BTRFS_LABEL_SIZE - 1] = '\0';
 948                         printk(KERN_INFO "BTRFS: device label %s ", disk_super->label);
 949                 } else {
 950                         printk(KERN_INFO "BTRFS: device fsid %pU ", disk_super->fsid);
 951                 }
 952
 953                 printk(KERN_CONT "devid %llu transid %llu %s\n", devid, transid, path);
 954                 ret = 0;
 955         }
 956         if (!ret && fs_devices_ret)
 957                 (*fs_devices_ret)->total_devices = total_devices;
 958
 959 error_unmap:
 960         kunmap(page);
 961         page_cache_release(page);
 962
 963 error_bdev_put:
 964         blkdev_put(bdev, flags);
 965 error:
 966         mutex_unlock(&uuid_mutex);
 967         return ret;
 968 }
 969
 970 /* helper to account the used device space in the range */
 971 int btrfs_account_dev_extents_size(struct btrfs_device *device, u64 start,
 972                                    u64 end, u64 *length)
 973 {
 974         struct btrfs_key key;
 975         struct btrfs_root *root = device->dev_root;
 976         struct btrfs_dev_extent *dev_extent;
 977         struct btrfs_path *path;
 978         u64 extent_end;
 979         int ret;
 980         int slot;
 981         struct extent_buffer *l;
 982
 983         *length = 0;
 984
 985         if (start >= device->total_bytes || device->is_tgtdev_for_dev_replace)
 986                 return 0;
 987
 988         path = btrfs_alloc_path();
 989         if (!path)
 990                 return -ENOMEM;
 991         path->reada = 2;
 992
 993         key.objectid = device->devid;
 994         key.offset = start;
 995         key.type = BTRFS_DEV_EXTENT_KEY;
 996
 997         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 998         if (ret < 0)
 999                 goto out;
1000         if (ret > 0) {
1001                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1002                 if (ret < 0)
1003                         goto out;
1004         }
1005
1006         while (1) {
1007                 l = path->nodes[0];
1008                 slot = path->slots[0];
1009                 if (slot >= btrfs_header_nritems(l)) {
1010                         ret = btrfs_next_leaf(root, path);
1011                         if (ret == 0)
1012                                 continue;
1013                         if (ret < 0)
1014                                 goto out;
1015
1016                         break;
1017                 }
1018                 btrfs_item_key_to_cpu(l, &key, slot);
1019
1020                 if (key.objectid < device->devid)
1021                         goto next;
1022
1023                 if (key.objectid > device->devid)
1024                         break;
1025
1026                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1027                         goto next;
1028
1029                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1030                 extent_end = key.offset + btrfs_dev_extent_length(l,
1031                                                                   dev_extent);
1032                 if (key.offset <= start && extent_end > end) {
1033                         *length = end - start + 1;
1034                         break;
1035                 } else if (key.offset <= start && extent_end > start)
1036                         *length += extent_end - start;
1037                 else if (key.offset > start && extent_end <= end)
1038                         *length += extent_end - key.offset;
1039                 else if (key.offset > start && key.offset <= end) {
1040                         *length += end - key.offset + 1;
1041                         break;
1042                 } else if (key.offset > end)
1043                         break;
1044
1045 next:
1046                 path->slots[0]++;
1047         }
1048         ret = 0;
1049 out:
1050         btrfs_free_path(path);
1051         return ret;
1052 }
1053
1054 static int contains_pending_extent(struct btrfs_trans_handle *trans,
1055                                    struct btrfs_device *device,
1056                                    u64 *start, u64 len)
1057 {
1058         struct extent_map *em;
1059         struct list_head *search_list = &trans->transaction->pending_chunks;
1060         int ret = 0;
1061         u64 physical_start = *start;
1062
1063 again:
1064         list_for_each_entry(em, search_list, list) {
1065                 struct map_lookup *map;
1066                 int i;
1067
1068                 map = (struct map_lookup *)em->bdev;
1069                 for (i = 0; i < map->num_stripes; i++) {
1070                         if (map->stripes[i].dev != device)
1071                                 continue;
1072                         if (map->stripes[i].physical >= physical_start + len ||
1073                             map->stripes[i].physical + em->orig_block_len <=
1074                             physical_start)
1075                                 continue;
1076                         *start = map->stripes[i].physical +
1077                                 em->orig_block_len;
1078                         ret = 1;
1079                 }
1080         }
1081         if (search_list == &trans->transaction->pending_chunks) {
1082                 search_list = &trans->root->fs_info->pinned_chunks;
1083                 goto again;
1084         }
1085
1086         return ret;
1087 }
1088
1089
1090 /*
1091  * find_free_dev_extent - find free space in the specified device
1092  * @device:     the device which we search the free space in
1093  * @num_bytes:  the size of the free space that we need
1094  * @start:      store the start of the free space.
1095  * @len:        the size of the free space. that we find, or the size of the max
1096  *              free space if we don't find suitable free space
1097  *
1098  * this uses a pretty simple search, the expectation is that it is
1099  * called very infrequently and that a given device has a small number
1100  * of extents
1101  *
1102  * @start is used to store the start of the free space if we find. But if we
1103  * don't find suitable free space, it will be used to store the start position
1104  * of the max free space.
1105  *
1106  * @len is used to store the size of the free space that we find.
1107  * But if we don't find suitable free space, it is used to store the size of
1108  * the max free space.
1109  */
1110 int find_free_dev_extent(struct btrfs_trans_handle *trans,
1111                          struct btrfs_device *device, u64 num_bytes,
1112                          u64 *start, u64 *len)
1113 {
1114         struct btrfs_key key;
1115         struct btrfs_root *root = device->dev_root;
1116         struct btrfs_dev_extent *dev_extent;
1117         struct btrfs_path *path;
1118         u64 hole_size;
1119         u64 max_hole_start;
1120         u64 max_hole_size;
1121         u64 extent_end;
1122         u64 search_start;
1123         u64 search_end = device->total_bytes;
1124         int ret;
1125         int slot;
1126         struct extent_buffer *l;
1127
1128         /* FIXME use last free of some kind */
1129
1130         /* we don't want to overwrite the superblock on the drive,
1131          * so we make sure to start at an offset of at least 1MB
1132          */
1133         search_start = max(root->fs_info->alloc_start, 1024ull * 1024);
1134
1135         path = btrfs_alloc_path();
1136         if (!path)
1137                 return -ENOMEM;
1138
1139         max_hole_start = search_start;
1140         max_hole_size = 0;
1141
1142 again:
1143         if (search_start >= search_end || device->is_tgtdev_for_dev_replace) {
1144                 ret = -ENOSPC;
1145                 goto out;
1146         }
1147
1148         path->reada = 2;
1149         path->search_commit_root = 1;
1150         path->skip_locking = 1;
1151
1152         key.objectid = device->devid;
1153         key.offset = search_start;
1154         key.type = BTRFS_DEV_EXTENT_KEY;
1155
1156         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1157         if (ret < 0)
1158                 goto out;
1159         if (ret > 0) {
1160                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1161                 if (ret < 0)
1162                         goto out;
1163         }
1164
1165         while (1) {
1166                 l = path->nodes[0];
1167                 slot = path->slots[0];
1168                 if (slot >= btrfs_header_nritems(l)) {
1169                         ret = btrfs_next_leaf(root, path);
1170                         if (ret == 0)
1171                                 continue;
1172                         if (ret < 0)
1173                                 goto out;
1174
1175                         break;
1176                 }
1177                 btrfs_item_key_to_cpu(l, &key, slot);
1178
1179                 if (key.objectid < device->devid)
1180                         goto next;
1181
1182                 if (key.objectid > device->devid)
1183                         break;
1184
1185                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1186                         goto next;
1187
1188                 if (key.offset > search_start) {
1189                         hole_size = key.offset - search_start;
1190
1191                         /*
1192                          * Have to check before we set max_hole_start, otherwise
1193                          * we could end up sending back this offset anyway.
1194                          */
1195                         if (contains_pending_extent(trans, device,
1196                                                     &search_start,
1197                                                     hole_size)) {
1198                                 if (key.offset >= search_start) {
1199                                         hole_size = key.offset - search_start;
1200                                 } else {
1201                                         WARN_ON_ONCE(1);
1202                                         hole_size = 0;
1203                                 }
1204                         }
1205
1206                         if (hole_size > max_hole_size) {
1207                                 max_hole_start = search_start;
1208                                 max_hole_size = hole_size;
1209                         }
1210
1211                         /*
1212                          * If this free space is greater than which we need,
1213                          * it must be the max free space that we have found
1214                          * until now, so max_hole_start must point to the start
1215                          * of this free space and the length of this free space
1216                          * is stored in max_hole_size. Thus, we return
1217                          * max_hole_start and max_hole_size and go back to the
1218                          * caller.
1219                          */
1220                         if (hole_size >= num_bytes) {
1221                                 ret = 0;
1222                                 goto out;
1223                         }
1224                 }
1225
1226                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1227                 extent_end = key.offset + btrfs_dev_extent_length(l,
1228                                                                   dev_extent);
1229                 if (extent_end > search_start)
1230                         search_start = extent_end;
1231 next:
1232                 path->slots[0]++;
1233                 cond_resched();
1234         }
1235
1236         /*
1237          * At this point, search_start should be the end of
1238          * allocated dev extents, and when shrinking the device,
1239          * search_end may be smaller than search_start.
1240          */
1241         if (search_end > search_start) {
1242                 hole_size = search_end - search_start;
1243
1244                 if (contains_pending_extent(trans, device, &search_start,
1245                                             hole_size)) {
1246                         btrfs_release_path(path);
1247                         goto again;
1248                 }
1249
1250                 if (hole_size > max_hole_size) {
1251                         max_hole_start = search_start;
1252                         max_hole_size = hole_size;
1253                 }
1254         }
1255
1256         /* See above. */
1257         if (max_hole_size < num_bytes)
1258                 ret = -ENOSPC;
1259         else
1260                 ret = 0;
1261
1262 out:
1263         btrfs_free_path(path);
1264         *start = max_hole_start;
1265         if (len)
1266                 *len = max_hole_size;
1267         return ret;
1268 }
1269
1270 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1271                           struct btrfs_device *device,
1272                           u64 start, u64 *dev_extent_len)
1273 {
1274         int ret;
1275         struct btrfs_path *path;
1276         struct btrfs_root *root = device->dev_root;
1277         struct btrfs_key key;
1278         struct btrfs_key found_key;
1279         struct extent_buffer *leaf = NULL;
1280         struct btrfs_dev_extent *extent = NULL;
1281
1282         path = btrfs_alloc_path();
1283         if (!path)
1284                 return -ENOMEM;
1285
1286         key.objectid = device->devid;
1287         key.offset = start;
1288         key.type = BTRFS_DEV_EXTENT_KEY;
1289 again:
1290         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1291         if (ret > 0) {
1292                 ret = btrfs_previous_item(root, path, key.objectid,
1293                                           BTRFS_DEV_EXTENT_KEY);
1294                 if (ret)
1295                         goto out;
1296                 leaf = path->nodes[0];
1297                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1298                 extent = btrfs_item_ptr(leaf, path->slots[0],
1299                                         struct btrfs_dev_extent);
1300                 BUG_ON(found_key.offset > start || found_key.offset +
1301                        btrfs_dev_extent_length(leaf, extent) < start);
1302                 key = found_key;
1303                 btrfs_release_path(path);
1304                 goto again;
1305         } else if (ret == 0) {
1306                 leaf = path->nodes[0];
1307                 extent = btrfs_item_ptr(leaf, path->slots[0],
1308                                         struct btrfs_dev_extent);
1309         } else {
1310                 btrfs_error(root->fs_info, ret, "Slot search failed");
1311                 goto out;
1312         }
1313
1314         *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1315
1316         ret = btrfs_del_item(trans, root, path);
1317         if (ret) {
1318                 btrfs_error(root->fs_info, ret,
1319                             "Failed to remove dev extent item");
1320         } else {
1321                 trans->transaction->have_free_bgs = 1;
1322         }
1323 out:
1324         btrfs_free_path(path);
1325         return ret;
1326 }
1327
1328 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1329                                   struct btrfs_device *device,
1330                                   u64 chunk_tree, u64 chunk_objectid,
1331                                   u64 chunk_offset, u64 start, u64 num_bytes)
1332 {
1333         int ret;
1334         struct btrfs_path *path;
1335         struct btrfs_root *root = device->dev_root;
1336         struct btrfs_dev_extent *extent;
1337         struct extent_buffer *leaf;
1338         struct btrfs_key key;
1339
1340         WARN_ON(!device->in_fs_metadata);
1341         WARN_ON(device->is_tgtdev_for_dev_replace);
1342         path = btrfs_alloc_path();
1343         if (!path)
1344                 return -ENOMEM;
1345
1346         key.objectid = device->devid;
1347         key.offset = start;
1348         key.type = BTRFS_DEV_EXTENT_KEY;
1349         ret = btrfs_insert_empty_item(trans, root, path, &key,
1350                                       sizeof(*extent));
1351         if (ret)
1352                 goto out;
1353
1354         leaf = path->nodes[0];
1355         extent = btrfs_item_ptr(leaf, path->slots[0],
1356                                 struct btrfs_dev_extent);
1357         btrfs_set_dev_extent_chunk_tree(leaf, extent, chunk_tree);
1358         btrfs_set_dev_extent_chunk_objectid(leaf, extent, chunk_objectid);
1359         btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1360
1361         write_extent_buffer(leaf, root->fs_info->chunk_tree_uuid,
1362                     btrfs_dev_extent_chunk_tree_uuid(extent), BTRFS_UUID_SIZE);
1363
1364         btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1365         btrfs_mark_buffer_dirty(leaf);
1366 out:
1367         btrfs_free_path(path);
1368         return ret;
1369 }
1370
1371 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1372 {
1373         struct extent_map_tree *em_tree;
1374         struct extent_map *em;
1375         struct rb_node *n;
1376         u64 ret = 0;
1377
1378         em_tree = &fs_info->mapping_tree.map_tree;
1379         read_lock(&em_tree->lock);
1380         n = rb_last(&em_tree->map);
1381         if (n) {
1382                 em = rb_entry(n, struct extent_map, rb_node);
1383                 ret = em->start + em->len;
1384         }
1385         read_unlock(&em_tree->lock);
1386
1387         return ret;
1388 }
1389
1390 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1391                                     u64 *devid_ret)
1392 {
1393         int ret;
1394         struct btrfs_key key;
1395         struct btrfs_key found_key;
1396         struct btrfs_path *path;
1397
1398         path = btrfs_alloc_path();
1399         if (!path)
1400                 return -ENOMEM;
1401
1402         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1403         key.type = BTRFS_DEV_ITEM_KEY;
1404         key.offset = (u64)-1;
1405
1406         ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1407         if (ret < 0)
1408                 goto error;
1409
1410         BUG_ON(ret == 0); /* Corruption */
1411
1412         ret = btrfs_previous_item(fs_info->chunk_root, path,
1413                                   BTRFS_DEV_ITEMS_OBJECTID,
1414                                   BTRFS_DEV_ITEM_KEY);
1415         if (ret) {
1416                 *devid_ret = 1;
1417         } else {
1418                 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1419                                       path->slots[0]);
1420                 *devid_ret = found_key.offset + 1;
1421         }
1422         ret = 0;
1423 error:
1424         btrfs_free_path(path);
1425         return ret;
1426 }
1427
1428 /*
1429  * the device information is stored in the chunk root
1430  * the btrfs_device struct should be fully filled in
1431  */
1432 static int btrfs_add_device(struct btrfs_trans_handle *trans,
1433                             struct btrfs_root *root,
1434                             struct btrfs_device *device)
1435 {
1436         int ret;
1437         struct btrfs_path *path;
1438         struct btrfs_dev_item *dev_item;
1439         struct extent_buffer *leaf;
1440         struct btrfs_key key;
1441         unsigned long ptr;
1442
1443         root = root->fs_info->chunk_root;
1444
1445         path = btrfs_alloc_path();
1446         if (!path)
1447                 return -ENOMEM;
1448
1449         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1450         key.type = BTRFS_DEV_ITEM_KEY;
1451         key.offset = device->devid;
1452
1453         ret = btrfs_insert_empty_item(trans, root, path, &key,
1454                                       sizeof(*dev_item));
1455         if (ret)
1456                 goto out;
1457
1458         leaf = path->nodes[0];
1459         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1460
1461         btrfs_set_device_id(leaf, dev_item, device->devid);
1462         btrfs_set_device_generation(leaf, dev_item, 0);
1463         btrfs_set_device_type(leaf, dev_item, device->type);
1464         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1465         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1466         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1467         btrfs_set_device_total_bytes(leaf, dev_item,
1468                                      btrfs_device_get_disk_total_bytes(device));
1469         btrfs_set_device_bytes_used(leaf, dev_item,
1470                                     btrfs_device_get_bytes_used(device));
1471         btrfs_set_device_group(leaf, dev_item, 0);
1472         btrfs_set_device_seek_speed(leaf, dev_item, 0);
1473         btrfs_set_device_bandwidth(leaf, dev_item, 0);
1474         btrfs_set_device_start_offset(leaf, dev_item, 0);
1475
1476         ptr = btrfs_device_uuid(dev_item);
1477         write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1478         ptr = btrfs_device_fsid(dev_item);
1479         write_extent_buffer(leaf, root->fs_info->fsid, ptr, BTRFS_UUID_SIZE);
1480         btrfs_mark_buffer_dirty(leaf);
1481
1482         ret = 0;
1483 out:
1484         btrfs_free_path(path);
1485         return ret;
1486 }
1487
1488 /*
1489  * Function to update ctime/mtime for a given device path.
1490  * Mainly used for ctime/mtime based probe like libblkid.
1491  */
1492 static void update_dev_time(char *path_name)
1493 {
1494         struct file *filp;
1495
1496         filp = filp_open(path_name, O_RDWR, 0);
1497         if (IS_ERR(filp))
1498                 return;
1499         file_update_time(filp);
1500         filp_close(filp, NULL);
1501         return;
1502 }
1503
1504 static int btrfs_rm_dev_item(struct btrfs_root *root,
1505                              struct btrfs_device *device)
1506 {
1507         int ret;
1508         struct btrfs_path *path;
1509         struct btrfs_key key;
1510         struct btrfs_trans_handle *trans;
1511
1512         root = root->fs_info->chunk_root;
1513
1514         path = btrfs_alloc_path();
1515         if (!path)
1516                 return -ENOMEM;
1517
1518         trans = btrfs_start_transaction(root, 0);
1519         if (IS_ERR(trans)) {
1520                 btrfs_free_path(path);
1521                 return PTR_ERR(trans);
1522         }
1523         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1524         key.type = BTRFS_DEV_ITEM_KEY;
1525         key.offset = device->devid;
1526
1527         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1528         if (ret < 0)
1529                 goto out;
1530
1531         if (ret > 0) {
1532                 ret = -ENOENT;
1533                 goto out;
1534         }
1535
1536         ret = btrfs_del_item(trans, root, path);
1537         if (ret)
1538                 goto out;
1539 out:
1540         btrfs_free_path(path);
1541         btrfs_commit_transaction(trans, root);
1542         return ret;
1543 }
1544
1545 int btrfs_rm_device(struct btrfs_root *root, char *device_path)
1546 {
1547         struct btrfs_device *device;
1548         struct btrfs_device *next_device;
1549         struct block_device *bdev;
1550         struct buffer_head *bh = NULL;
1551         struct btrfs_super_block *disk_super;
1552         struct btrfs_fs_devices *cur_devices;
1553         u64 all_avail;
1554         u64 devid;
1555         u64 num_devices;
1556         u8 *dev_uuid;
1557         unsigned seq;
1558         int ret = 0;
1559         bool clear_super = false;
1560
1561         mutex_lock(&uuid_mutex);
1562
1563         do {
1564                 seq = read_seqbegin(&root->fs_info->profiles_lock);
1565
1566                 all_avail = root->fs_info->avail_data_alloc_bits |
1567                             root->fs_info->avail_system_alloc_bits |
1568                             root->fs_info->avail_metadata_alloc_bits;
1569         } while (read_seqretry(&root->fs_info->profiles_lock, seq));
1570
1571         num_devices = root->fs_info->fs_devices->num_devices;
1572         btrfs_dev_replace_lock(&root->fs_info->dev_replace);
1573         if (btrfs_dev_replace_is_ongoing(&root->fs_info->dev_replace)) {
1574                 WARN_ON(num_devices < 1);
1575                 num_devices--;
1576         }
1577         btrfs_dev_replace_unlock(&root->fs_info->dev_replace);
1578
1579         if ((all_avail & BTRFS_BLOCK_GROUP_RAID10) && num_devices <= 4) {
1580                 ret = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET;
1581                 goto out;
1582         }
1583
1584         if ((all_avail & BTRFS_BLOCK_GROUP_RAID1) && num_devices <= 2) {
1585                 ret = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET;
1586                 goto out;
1587         }
1588
1589         if ((all_avail & BTRFS_BLOCK_GROUP_RAID5) &&
1590             root->fs_info->fs_devices->rw_devices <= 2) {
1591                 ret = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET;
1592                 goto out;
1593         }
1594         if ((all_avail & BTRFS_BLOCK_GROUP_RAID6) &&
1595             root->fs_info->fs_devices->rw_devices <= 3) {
1596                 ret = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET;
1597                 goto out;
1598         }
1599
1600         if (strcmp(device_path, "missing") == 0) {
1601                 struct list_head *devices;
1602                 struct btrfs_device *tmp;
1603
1604                 device = NULL;
1605                 devices = &root->fs_info->fs_devices->devices;
1606                 /*
1607                  * It is safe to read the devices since the volume_mutex
1608                  * is held.
1609                  */
1610                 list_for_each_entry(tmp, devices, dev_list) {
1611                         if (tmp->in_fs_metadata &&
1612                             !tmp->is_tgtdev_for_dev_replace &&
1613                             !tmp->bdev) {
1614                                 device = tmp;
1615                                 break;
1616                         }
1617                 }
1618                 bdev = NULL;
1619                 bh = NULL;
1620                 disk_super = NULL;
1621                 if (!device) {
1622                         ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
1623                         goto out;
1624                 }
1625         } else {
1626                 ret = btrfs_get_bdev_and_sb(device_path,
1627                                             FMODE_WRITE | FMODE_EXCL,
1628                                             root->fs_info->bdev_holder, 0,
1629                                             &bdev, &bh);
1630                 if (ret)
1631                         goto out;
1632                 disk_super = (struct btrfs_super_block *)bh->b_data;
1633                 devid = btrfs_stack_device_id(&disk_super->dev_item);
1634                 dev_uuid = disk_super->dev_item.uuid;
1635                 device = btrfs_find_device(root->fs_info, devid, dev_uuid,
1636                                            disk_super->fsid);
1637                 if (!device) {
1638                         ret = -ENOENT;
1639                         goto error_brelse;
1640                 }
1641         }
1642
1643         if (device->is_tgtdev_for_dev_replace) {
1644                 ret = BTRFS_ERROR_DEV_TGT_REPLACE;
1645                 goto error_brelse;
1646         }
1647
1648         if (device->writeable && root->fs_info->fs_devices->rw_devices == 1) {
1649                 ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
1650                 goto error_brelse;
1651         }
1652
1653         if (device->writeable) {
1654                 lock_chunks(root);
1655                 list_del_init(&device->dev_alloc_list);
1656                 device->fs_devices->rw_devices--;
1657                 unlock_chunks(root);
1658                 clear_super = true;
1659         }
1660
1661         mutex_unlock(&uuid_mutex);
1662         ret = btrfs_shrink_device(device, 0);
1663         mutex_lock(&uuid_mutex);
1664         if (ret)
1665                 goto error_undo;
1666
1667         /*
1668          * TODO: the superblock still includes this device in its num_devices
1669          * counter although write_all_supers() is not locked out. This
1670          * could give a filesystem state which requires a degraded mount.
1671          */
1672         ret = btrfs_rm_dev_item(root->fs_info->chunk_root, device);
1673         if (ret)
1674                 goto error_undo;
1675
1676         device->in_fs_metadata = 0;
1677         btrfs_scrub_cancel_dev(root->fs_info, device);
1678
1679         /*
1680          * the device list mutex makes sure that we don't change
1681          * the device list while someone else is writing out all
1682          * the device supers. Whoever is writing all supers, should
1683          * lock the device list mutex before getting the number of
1684          * devices in the super block (super_copy). Conversely,
1685          * whoever updates the number of devices in the super block
1686          * (super_copy) should hold the device list mutex.
1687          */
1688
1689         cur_devices = device->fs_devices;
1690         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1691         list_del_rcu(&device->dev_list);
1692
1693         device->fs_devices->num_devices--;
1694         device->fs_devices->total_devices--;
1695
1696         if (device->missing)
1697                 device->fs_devices->missing_devices--;
1698
1699         next_device = list_entry(root->fs_info->fs_devices->devices.next,
1700                                  struct btrfs_device, dev_list);
1701         if (device->bdev == root->fs_info->sb->s_bdev)
1702                 root->fs_info->sb->s_bdev = next_device->bdev;
1703         if (device->bdev == root->fs_info->fs_devices->latest_bdev)
1704                 root->fs_info->fs_devices->latest_bdev = next_device->bdev;
1705
1706         if (device->bdev) {
1707                 device->fs_devices->open_devices--;
1708                 /* remove sysfs entry */
1709                 btrfs_kobj_rm_device(root->fs_info, device);
1710         }
1711
1712         call_rcu(&device->rcu, free_device);
1713
1714         num_devices = btrfs_super_num_devices(root->fs_info->super_copy) - 1;
1715         btrfs_set_super_num_devices(root->fs_info->super_copy, num_devices);
1716         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1717
1718         if (cur_devices->open_devices == 0) {
1719                 struct btrfs_fs_devices *fs_devices;
1720                 fs_devices = root->fs_info->fs_devices;
1721                 while (fs_devices) {
1722                         if (fs_devices->seed == cur_devices) {
1723                                 fs_devices->seed = cur_devices->seed;
1724                                 break;
1725                         }
1726                         fs_devices = fs_devices->seed;
1727                 }
1728                 cur_devices->seed = NULL;
1729                 __btrfs_close_devices(cur_devices);
1730                 free_fs_devices(cur_devices);
1731         }
1732
1733         root->fs_info->num_tolerated_disk_barrier_failures =
1734                 btrfs_calc_num_tolerated_disk_barrier_failures(root->fs_info);
1735
1736         /*
1737          * at this point, the device is zero sized.  We want to
1738          * remove it from the devices list and zero out the old super
1739          */
1740         if (clear_super && disk_super) {
1741                 u64 bytenr;
1742                 int i;
1743
1744                 /* make sure this device isn't detected as part of
1745                  * the FS anymore
1746                  */
1747                 memset(&disk_super->magic, 0, sizeof(disk_super->magic));
1748                 set_buffer_dirty(bh);
1749                 sync_dirty_buffer(bh);
1750
1751                 /* clear the mirror copies of super block on the disk
1752                  * being removed, 0th copy is been taken care above and
1753                  * the below would take of the rest
1754                  */
1755                 for (i = 1; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1756                         bytenr = btrfs_sb_offset(i);
1757                         if (bytenr + BTRFS_SUPER_INFO_SIZE >=
1758                                         i_size_read(bdev->bd_inode))
1759                                 break;
1760
1761                         brelse(bh);
1762                         bh = __bread(bdev, bytenr / 4096,
1763                                         BTRFS_SUPER_INFO_SIZE);
1764                         if (!bh)
1765                                 continue;
1766
1767                         disk_super = (struct btrfs_super_block *)bh->b_data;
1768
1769                         if (btrfs_super_bytenr(disk_super) != bytenr ||
1770                                 btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
1771                                 continue;
1772                         }
1773                         memset(&disk_super->magic, 0,
1774                                                 sizeof(disk_super->magic));
1775                         set_buffer_dirty(bh);
1776                         sync_dirty_buffer(bh);
1777                 }
1778         }
1779
1780         ret = 0;
1781
1782         if (bdev) {
1783                 /* Notify udev that device has changed */
1784                 btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
1785
1786                 /* Update ctime/mtime for device path for libblkid */
1787                 update_dev_time(device_path);
1788         }
1789
1790 error_brelse:
1791         brelse(bh);
1792         if (bdev)
1793                 blkdev_put(bdev, FMODE_READ | FMODE_EXCL);
1794 out:
1795         mutex_unlock(&uuid_mutex);
1796         return ret;
1797 error_undo:
1798         if (device->writeable) {
1799                 lock_chunks(root);
1800                 list_add(&device->dev_alloc_list,
1801                          &root->fs_info->fs_devices->alloc_list);
1802                 device->fs_devices->rw_devices++;
1803                 unlock_chunks(root);
1804         }
1805         goto error_brelse;
1806 }
1807
1808 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_fs_info *fs_info,
1809                                         struct btrfs_device *srcdev)
1810 {
1811         struct btrfs_fs_devices *fs_devices;
1812
1813         WARN_ON(!mutex_is_locked(&fs_info->fs_devices->device_list_mutex));
1814
1815         /*
1816          * in case of fs with no seed, srcdev->fs_devices will point
1817          * to fs_devices of fs_info. However when the dev being replaced is
1818          * a seed dev it will point to the seed's local fs_devices. In short
1819          * srcdev will have its correct fs_devices in both the cases.
1820          */
1821         fs_devices = srcdev->fs_devices;
1822
1823         list_del_rcu(&srcdev->dev_list);
1824         list_del_rcu(&srcdev->dev_alloc_list);
1825         fs_devices->num_devices--;
1826         if (srcdev->missing)
1827                 fs_devices->missing_devices--;
1828
1829         if (srcdev->writeable) {
1830                 fs_devices->rw_devices--;
1831                 /* zero out the old super if it is writable */
1832                 btrfs_scratch_superblock(srcdev);
1833         }
1834
1835         if (srcdev->bdev)
1836                 fs_devices->open_devices--;
1837 }
1838
1839 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_fs_info *fs_info,
1840                                       struct btrfs_device *srcdev)
1841 {
1842         struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
1843
1844         call_rcu(&srcdev->rcu, free_device);
1845
1846         /*
1847          * unless fs_devices is seed fs, num_devices shouldn't go
1848          * zero
1849          */
1850         BUG_ON(!fs_devices->num_devices && !fs_devices->seeding);
1851
1852         /* if this is no devs we rather delete the fs_devices */
1853         if (!fs_devices->num_devices) {
1854                 struct btrfs_fs_devices *tmp_fs_devices;
1855
1856                 tmp_fs_devices = fs_info->fs_devices;
1857                 while (tmp_fs_devices) {
1858                         if (tmp_fs_devices->seed == fs_devices) {
1859                                 tmp_fs_devices->seed = fs_devices->seed;
1860                                 break;
1861                         }
1862                         tmp_fs_devices = tmp_fs_devices->seed;
1863                 }
1864                 fs_devices->seed = NULL;
1865                 __btrfs_close_devices(fs_devices);
1866                 free_fs_devices(fs_devices);
1867         }
1868 }
1869
1870 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
1871                                       struct btrfs_device *tgtdev)
1872 {
1873         struct btrfs_device *next_device;
1874
1875         mutex_lock(&uuid_mutex);
1876         WARN_ON(!tgtdev);
1877         mutex_lock(&fs_info->fs_devices->device_list_mutex);
1878         if (tgtdev->bdev) {
1879                 btrfs_scratch_superblock(tgtdev);
1880                 fs_info->fs_devices->open_devices--;
1881         }
1882         fs_info->fs_devices->num_devices--;
1883
1884         next_device = list_entry(fs_info->fs_devices->devices.next,
1885                                  struct btrfs_device, dev_list);
1886         if (tgtdev->bdev == fs_info->sb->s_bdev)
1887                 fs_info->sb->s_bdev = next_device->bdev;
1888         if (tgtdev->bdev == fs_info->fs_devices->latest_bdev)
1889                 fs_info->fs_devices->latest_bdev = next_device->bdev;
1890         list_del_rcu(&tgtdev->dev_list);
1891
1892         call_rcu(&tgtdev->rcu, free_device);
1893
1894         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
1895         mutex_unlock(&uuid_mutex);
1896 }
1897
1898 static int btrfs_find_device_by_path(struct btrfs_root *root, char *device_path,
1899                                      struct btrfs_device **device)
1900 {
1901         int ret = 0;
1902         struct btrfs_super_block *disk_super;
1903         u64 devid;
1904         u8 *dev_uuid;
1905         struct block_device *bdev;
1906         struct buffer_head *bh;
1907
1908         *device = NULL;
1909         ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
1910                                     root->fs_info->bdev_holder, 0, &bdev, &bh);
1911         if (ret)
1912                 return ret;
1913         disk_super = (struct btrfs_super_block *)bh->b_data;
1914         devid = btrfs_stack_device_id(&disk_super->dev_item);
1915         dev_uuid = disk_super->dev_item.uuid;
1916         *device = btrfs_find_device(root->fs_info, devid, dev_uuid,
1917                                     disk_super->fsid);
1918         brelse(bh);
1919         if (!*device)
1920                 ret = -ENOENT;
1921         blkdev_put(bdev, FMODE_READ);
1922         return ret;
1923 }
1924
1925 int btrfs_find_device_missing_or_by_path(struct btrfs_root *root,
1926                                          char *device_path,
1927                                          struct btrfs_device **device)
1928 {
1929         *device = NULL;
1930         if (strcmp(device_path, "missing") == 0) {
1931                 struct list_head *devices;
1932                 struct btrfs_device *tmp;
1933
1934                 devices = &root->fs_info->fs_devices->devices;
1935                 /*
1936                  * It is safe to read the devices since the volume_mutex
1937                  * is held by the caller.
1938                  */
1939                 list_for_each_entry(tmp, devices, dev_list) {
1940                         if (tmp->in_fs_metadata && !tmp->bdev) {
1941                                 *device = tmp;
1942                                 break;
1943                         }
1944                 }
1945
1946                 if (!*device) {
1947                         btrfs_err(root->fs_info, "no missing device found");
1948                         return -ENOENT;
1949                 }
1950
1951                 return 0;
1952         } else {
1953                 return btrfs_find_device_by_path(root, device_path, device);
1954         }
1955 }
1956
1957 /*
1958  * does all the dirty work required for changing file system's UUID.
1959  */
1960 static int btrfs_prepare_sprout(struct btrfs_root *root)
1961 {
1962         struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
1963         struct btrfs_fs_devices *old_devices;
1964         struct btrfs_fs_devices *seed_devices;
1965         struct btrfs_super_block *disk_super = root->fs_info->super_copy;
1966         struct btrfs_device *device;
1967         u64 super_flags;
1968
1969         BUG_ON(!mutex_is_locked(&uuid_mutex));
1970         if (!fs_devices->seeding)
1971                 return -EINVAL;
1972
1973         seed_devices = __alloc_fs_devices();
1974         if (IS_ERR(seed_devices))
1975                 return PTR_ERR(seed_devices);
1976
1977         old_devices = clone_fs_devices(fs_devices);
1978         if (IS_ERR(old_devices)) {
1979                 kfree(seed_devices);
1980                 return PTR_ERR(old_devices);
1981         }
1982
1983         list_add(&old_devices->list, &fs_uuids);
1984
1985         memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
1986         seed_devices->opened = 1;
1987         INIT_LIST_HEAD(&seed_devices->devices);
1988         INIT_LIST_HEAD(&seed_devices->alloc_list);
1989         mutex_init(&seed_devices->device_list_mutex);
1990
1991         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1992         list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
1993                               synchronize_rcu);
1994         list_for_each_entry(device, &seed_devices->devices, dev_list)
1995                 device->fs_devices = seed_devices;
1996
1997         lock_chunks(root);
1998         list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
1999         unlock_chunks(root);
2000
2001         fs_devices->seeding = 0;
2002         fs_devices->num_devices = 0;
2003         fs_devices->open_devices = 0;
2004         fs_devices->missing_devices = 0;
2005         fs_devices->rotating = 0;
2006         fs_devices->seed = seed_devices;
2007
2008         generate_random_uuid(fs_devices->fsid);
2009         memcpy(root->fs_info->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2010         memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2011         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
2012
2013         super_flags = btrfs_super_flags(disk_super) &
2014                       ~BTRFS_SUPER_FLAG_SEEDING;
2015         btrfs_set_super_flags(disk_super, super_flags);
2016
2017         return 0;
2018 }
2019
2020 /*
2021  * strore the expected generation for seed devices in device items.
2022  */
2023 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans,
2024                                struct btrfs_root *root)
2025 {
2026         struct btrfs_path *path;
2027         struct extent_buffer *leaf;
2028         struct btrfs_dev_item *dev_item;
2029         struct btrfs_device *device;
2030         struct btrfs_key key;
2031         u8 fs_uuid[BTRFS_UUID_SIZE];
2032         u8 dev_uuid[BTRFS_UUID_SIZE];
2033         u64 devid;
2034         int ret;
2035
2036         path = btrfs_alloc_path();
2037         if (!path)
2038                 return -ENOMEM;
2039
2040         root = root->fs_info->chunk_root;
2041         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2042         key.offset = 0;
2043         key.type = BTRFS_DEV_ITEM_KEY;
2044
2045         while (1) {
2046                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2047                 if (ret < 0)
2048                         goto error;
2049
2050                 leaf = path->nodes[0];
2051 next_slot:
2052                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2053                         ret = btrfs_next_leaf(root, path);
2054                         if (ret > 0)
2055                                 break;
2056                         if (ret < 0)
2057                                 goto error;
2058                         leaf = path->nodes[0];
2059                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2060                         btrfs_release_path(path);
2061                         continue;
2062                 }
2063
2064                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2065                 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2066                     key.type != BTRFS_DEV_ITEM_KEY)
2067                         break;
2068
2069                 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2070                                           struct btrfs_dev_item);
2071                 devid = btrfs_device_id(leaf, dev_item);
2072                 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2073                                    BTRFS_UUID_SIZE);
2074                 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2075                                    BTRFS_UUID_SIZE);
2076                 device = btrfs_find_device(root->fs_info, devid, dev_uuid,
2077                                            fs_uuid);
2078                 BUG_ON(!device); /* Logic error */
2079
2080                 if (device->fs_devices->seeding) {
2081                         btrfs_set_device_generation(leaf, dev_item,
2082                                                     device->generation);
2083                         btrfs_mark_buffer_dirty(leaf);
2084                 }
2085
2086                 path->slots[0]++;
2087                 goto next_slot;
2088         }
2089         ret = 0;
2090 error:
2091         btrfs_free_path(path);
2092         return ret;
2093 }
2094
2095 int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
2096 {
2097         struct request_queue *q;
2098         struct btrfs_trans_handle *trans;
2099         struct btrfs_device *device;
2100         struct block_device *bdev;
2101         struct list_head *devices;
2102         struct super_block *sb = root->fs_info->sb;
2103         struct rcu_string *name;
2104         u64 tmp;
2105         int seeding_dev = 0;
2106         int ret = 0;
2107
2108         if ((sb->s_flags & MS_RDONLY) && !root->fs_info->fs_devices->seeding)
2109                 return -EROFS;
2110
2111         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2112                                   root->fs_info->bdev_holder);
2113         if (IS_ERR(bdev))
2114                 return PTR_ERR(bdev);
2115
2116         if (root->fs_info->fs_devices->seeding) {
2117                 seeding_dev = 1;
2118                 down_write(&sb->s_umount);
2119                 mutex_lock(&uuid_mutex);
2120         }
2121
2122         filemap_write_and_wait(bdev->bd_inode->i_mapping);
2123
2124         devices = &root->fs_info->fs_devices->devices;
2125
2126         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
2127         list_for_each_entry(device, devices, dev_list) {
2128                 if (device->bdev == bdev) {
2129                         ret = -EEXIST;
2130                         mutex_unlock(
2131                                 &root->fs_info->fs_devices->device_list_mutex);
2132                         goto error;
2133                 }
2134         }
2135         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
2136
2137         device = btrfs_alloc_device(root->fs_info, NULL, NULL);
2138         if (IS_ERR(device)) {
2139                 /* we can safely leave the fs_devices entry around */
2140                 ret = PTR_ERR(device);
2141                 goto error;
2142         }
2143
2144         name = rcu_string_strdup(device_path, GFP_NOFS);
2145         if (!name) {
2146                 kfree(device);
2147                 ret = -ENOMEM;
2148                 goto error;
2149         }
2150         rcu_assign_pointer(device->name, name);
2151
2152         trans = btrfs_start_transaction(root, 0);
2153         if (IS_ERR(trans)) {
2154                 rcu_string_free(device->name);
2155                 kfree(device);
2156                 ret = PTR_ERR(trans);
2157                 goto error;
2158         }
2159
2160         q = bdev_get_queue(bdev);
2161         if (blk_queue_discard(q))
2162                 device->can_discard = 1;
2163         device->writeable = 1;
2164         device->generation = trans->transid;
2165         device->io_width = root->sectorsize;
2166         device->io_align = root->sectorsize;
2167         device->sector_size = root->sectorsize;
2168         device->total_bytes = i_size_read(bdev->bd_inode);
2169         device->disk_total_bytes = device->total_bytes;
2170         device->commit_total_bytes = device->total_bytes;
2171         device->dev_root = root->fs_info->dev_root;
2172         device->bdev = bdev;
2173         device->in_fs_metadata = 1;
2174         device->is_tgtdev_for_dev_replace = 0;
2175         device->mode = FMODE_EXCL;
2176         device->dev_stats_valid = 1;
2177         set_blocksize(device->bdev, 4096);
2178
2179         if (seeding_dev) {
2180                 sb->s_flags &= ~MS_RDONLY;
2181                 ret = btrfs_prepare_sprout(root);
2182                 BUG_ON(ret); /* -ENOMEM */
2183         }
2184
2185         device->fs_devices = root->fs_info->fs_devices;
2186
2187         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
2188         lock_chunks(root);
2189         list_add_rcu(&device->dev_list, &root->fs_info->fs_devices->devices);
2190         list_add(&device->dev_alloc_list,
2191                  &root->fs_info->fs_devices->alloc_list);
2192         root->fs_info->fs_devices->num_devices++;
2193         root->fs_info->fs_devices->open_devices++;
2194         root->fs_info->fs_devices->rw_devices++;
2195         root->fs_info->fs_devices->total_devices++;
2196         root->fs_info->fs_devices->total_rw_bytes += device->total_bytes;
2197
2198         spin_lock(&root->fs_info->free_chunk_lock);
2199         root->fs_info->free_chunk_space += device->total_bytes;
2200         spin_unlock(&root->fs_info->free_chunk_lock);
2201
2202         if (!blk_queue_nonrot(bdev_get_queue(bdev)))
2203                 root->fs_info->fs_devices->rotating = 1;
2204
2205         tmp = btrfs_super_total_bytes(root->fs_info->super_copy);
2206         btrfs_set_super_total_bytes(root->fs_info->super_copy,
2207                                     tmp + device->total_bytes);
2208
2209         tmp = btrfs_super_num_devices(root->fs_info->super_copy);
2210         btrfs_set_super_num_devices(root->fs_info->super_copy,
2211                                     tmp + 1);
2212
2213         /* add sysfs device entry */
2214         btrfs_kobj_add_device(root->fs_info, device);
2215
2216         /*
2217          * we've got more storage, clear any full flags on the space
2218          * infos
2219          */
2220         btrfs_clear_space_info_full(root->fs_info);
2221
2222         unlock_chunks(root);
2223         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
2224
2225         if (seeding_dev) {
2226                 lock_chunks(root);
2227                 ret = init_first_rw_device(trans, root, device);
2228                 unlock_chunks(root);
2229                 if (ret) {
2230                         btrfs_abort_transaction(trans, root, ret);
2231                         goto error_trans;
2232                 }
2233         }
2234
2235         ret = btrfs_add_device(trans, root, device);
2236         if (ret) {
2237                 btrfs_abort_transaction(trans, root, ret);
2238                 goto error_trans;
2239         }
2240
2241         if (seeding_dev) {
2242                 char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
2243
2244                 ret = btrfs_finish_sprout(trans, root);
2245                 if (ret) {
2246                         btrfs_abort_transaction(trans, root, ret);
2247                         goto error_trans;
2248                 }
2249
2250                 /* Sprouting would change fsid of the mounted root,
2251                  * so rename the fsid on the sysfs
2252                  */
2253                 snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
2254                                                 root->fs_info->fsid);
2255                 if (kobject_rename(&root->fs_info->super_kobj, fsid_buf))
2256                         goto error_trans;
2257         }
2258
2259         root->fs_info->num_tolerated_disk_barrier_failures =
2260                 btrfs_calc_num_tolerated_disk_barrier_failures(root->fs_info);
2261         ret = btrfs_commit_transaction(trans, root);
2262
2263         if (seeding_dev) {
2264                 mutex_unlock(&uuid_mutex);
2265                 up_write(&sb->s_umount);
2266
2267                 if (ret) /* transaction commit */
2268                         return ret;
2269
2270                 ret = btrfs_relocate_sys_chunks(root);
2271                 if (ret < 0)
2272                         btrfs_error(root->fs_info, ret,
2273                                     "Failed to relocate sys chunks after "
2274                                     "device initialization. This can be fixed "
2275                                     "using the \"btrfs balance\" command.");
2276                 trans = btrfs_attach_transaction(root);
2277                 if (IS_ERR(trans)) {
2278                         if (PTR_ERR(trans) == -ENOENT)
2279                                 return 0;
2280                         return PTR_ERR(trans);
2281                 }
2282                 ret = btrfs_commit_transaction(trans, root);
2283         }
2284
2285         /* Update ctime/mtime for libblkid */
2286         update_dev_time(device_path);
2287         return ret;
2288
2289 error_trans:
2290         btrfs_end_transaction(trans, root);
2291         rcu_string_free(device->name);
2292         btrfs_kobj_rm_device(root->fs_info, device);
2293         kfree(device);
2294 error:
2295         blkdev_put(bdev, FMODE_EXCL);
2296         if (seeding_dev) {
2297                 mutex_unlock(&uuid_mutex);
2298                 up_write(&sb->s_umount);
2299         }
2300         return ret;
2301 }
2302
2303 int btrfs_init_dev_replace_tgtdev(struct btrfs_root *root, char *device_path,
2304                                   struct btrfs_device *srcdev,
2305                                   struct btrfs_device **device_out)
2306 {
2307         struct request_queue *q;
2308         struct btrfs_device *device;
2309         struct block_device *bdev;
2310         struct btrfs_fs_info *fs_info = root->fs_info;
2311         struct list_head *devices;
2312         struct rcu_string *name;
2313         u64 devid = BTRFS_DEV_REPLACE_DEVID;
2314         int ret = 0;
2315
2316         *device_out = NULL;
2317         if (fs_info->fs_devices->seeding) {
2318                 btrfs_err(fs_info, "the filesystem is a seed filesystem!");
2319                 return -EINVAL;
2320         }
2321
2322         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2323                                   fs_info->bdev_holder);
2324         if (IS_ERR(bdev)) {
2325                 btrfs_err(fs_info, "target device %s is invalid!", device_path);
2326                 return PTR_ERR(bdev);
2327         }
2328
2329         filemap_write_and_wait(bdev->bd_inode->i_mapping);
2330
2331         devices = &fs_info->fs_devices->devices;
2332         list_for_each_entry(device, devices, dev_list) {
2333                 if (device->bdev == bdev) {
2334                         btrfs_err(fs_info, "target device is in the filesystem!");
2335                         ret = -EEXIST;
2336                         goto error;
2337                 }
2338         }
2339
2340
2341         if (i_size_read(bdev->bd_inode) <
2342             btrfs_device_get_total_bytes(srcdev)) {
2343                 btrfs_err(fs_info, "target device is smaller than source device!");
2344                 ret = -EINVAL;
2345                 goto error;
2346         }
2347
2348
2349         device = btrfs_alloc_device(NULL, &devid, NULL);
2350         if (IS_ERR(device)) {
2351                 ret = PTR_ERR(device);
2352                 goto error;
2353         }
2354
2355         name = rcu_string_strdup(device_path, GFP_NOFS);
2356         if (!name) {
2357                 kfree(device);
2358                 ret = -ENOMEM;
2359                 goto error;
2360         }
2361         rcu_assign_pointer(device->name, name);
2362
2363         q = bdev_get_queue(bdev);
2364         if (blk_queue_discard(q))
2365                 device->can_discard = 1;
2366         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
2367         device->writeable = 1;
2368         device->generation = 0;
2369         device->io_width = root->sectorsize;
2370         device->io_align = root->sectorsize;
2371         device->sector_size = root->sectorsize;
2372         device->total_bytes = btrfs_device_get_total_bytes(srcdev);
2373         device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
2374         device->bytes_used = btrfs_device_get_bytes_used(srcdev);
2375         ASSERT(list_empty(&srcdev->resized_list));
2376         device->commit_total_bytes = srcdev->commit_total_bytes;
2377         device->commit_bytes_used = device->bytes_used;
2378         device->dev_root = fs_info->dev_root;
2379         device->bdev = bdev;
2380         device->in_fs_metadata = 1;
2381         device->is_tgtdev_for_dev_replace = 1;
2382         device->mode = FMODE_EXCL;
2383         device->dev_stats_valid = 1;
2384         set_blocksize(device->bdev, 4096);
2385         device->fs_devices = fs_info->fs_devices;
2386         list_add(&device->dev_list, &fs_info->fs_devices->devices);
2387         fs_info->fs_devices->num_devices++;
2388         fs_info->fs_devices->open_devices++;
2389         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
2390
2391         *device_out = device;
2392         return ret;
2393
2394 error:
2395         blkdev_put(bdev, FMODE_EXCL);
2396         return ret;
2397 }
2398
2399 void btrfs_init_dev_replace_tgtdev_for_resume(struct btrfs_fs_info *fs_info,
2400                                               struct btrfs_device *tgtdev)
2401 {
2402         WARN_ON(fs_info->fs_devices->rw_devices == 0);
2403         tgtdev->io_width = fs_info->dev_root->sectorsize;
2404         tgtdev->io_align = fs_info->dev_root->sectorsize;
2405         tgtdev->sector_size = fs_info->dev_root->sectorsize;
2406         tgtdev->dev_root = fs_info->dev_root;
2407         tgtdev->in_fs_metadata = 1;
2408 }
2409
2410 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2411                                         struct btrfs_device *device)
2412 {
2413         int ret;
2414         struct btrfs_path *path;
2415         struct btrfs_root *root;
2416         struct btrfs_dev_item *dev_item;
2417         struct extent_buffer *leaf;
2418         struct btrfs_key key;
2419
2420         root = device->dev_root->fs_info->chunk_root;
2421
2422         path = btrfs_alloc_path();
2423         if (!path)
2424                 return -ENOMEM;
2425
2426         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2427         key.type = BTRFS_DEV_ITEM_KEY;
2428         key.offset = device->devid;
2429
2430         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2431         if (ret < 0)
2432                 goto out;
2433
2434         if (ret > 0) {
2435                 ret = -ENOENT;
2436                 goto out;
2437         }
2438
2439         leaf = path->nodes[0];
2440         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2441
2442         btrfs_set_device_id(leaf, dev_item, device->devid);
2443         btrfs_set_device_type(leaf, dev_item, device->type);
2444         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2445         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2446         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2447         btrfs_set_device_total_bytes(leaf, dev_item,
2448                                      btrfs_device_get_disk_total_bytes(device));
2449         btrfs_set_device_bytes_used(leaf, dev_item,
2450                                     btrfs_device_get_bytes_used(device));
2451         btrfs_mark_buffer_dirty(leaf);
2452
2453 out:
2454         btrfs_free_path(path);
2455         return ret;
2456 }
2457
2458 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2459                       struct btrfs_device *device, u64 new_size)
2460 {
2461         struct btrfs_super_block *super_copy =
2462                 device->dev_root->fs_info->super_copy;
2463         struct btrfs_fs_devices *fs_devices;
2464         u64 old_total;
2465         u64 diff;
2466
2467         if (!device->writeable)
2468                 return -EACCES;
2469
2470         lock_chunks(device->dev_root);
2471         old_total = btrfs_super_total_bytes(super_copy);
2472         diff = new_size - device->total_bytes;
2473
2474         if (new_size <= device->total_bytes ||
2475             device->is_tgtdev_for_dev_replace) {
2476                 unlock_chunks(device->dev_root);
2477                 return -EINVAL;
2478         }
2479
2480         fs_devices = device->dev_root->fs_info->fs_devices;
2481
2482         btrfs_set_super_total_bytes(super_copy, old_total + diff);
2483         device->fs_devices->total_rw_bytes += diff;
2484
2485         btrfs_device_set_total_bytes(device, new_size);
2486         btrfs_device_set_disk_total_bytes(device, new_size);
2487         btrfs_clear_space_info_full(device->dev_root->fs_info);
2488         if (list_empty(&device->resized_list))
2489                 list_add_tail(&device->resized_list,
2490                               &fs_devices->resized_devices);
2491         unlock_chunks(device->dev_root);
2492
2493         return btrfs_update_device(trans, device);
2494 }
2495
2496 static int btrfs_free_chunk(struct btrfs_trans_handle *trans,
2497                             struct btrfs_root *root, u64 chunk_objectid,
2498                             u64 chunk_offset)
2499 {
2500         int ret;
2501         struct btrfs_path *path;
2502         struct btrfs_key key;
2503
2504         root = root->fs_info->chunk_root;
2505         path = btrfs_alloc_path();
2506         if (!path)
2507                 return -ENOMEM;
2508
2509         key.objectid = chunk_objectid;
2510         key.offset = chunk_offset;
2511         key.type = BTRFS_CHUNK_ITEM_KEY;
2512
2513         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2514         if (ret < 0)
2515                 goto out;
2516         else if (ret > 0) { /* Logic error or corruption */
2517                 btrfs_error(root->fs_info, -ENOENT,
2518                             "Failed lookup while freeing chunk.");
2519                 ret = -ENOENT;
2520                 goto out;
2521         }
2522
2523         ret = btrfs_del_item(trans, root, path);
2524         if (ret < 0)
2525                 btrfs_error(root->fs_info, ret,
2526                             "Failed to delete chunk item.");
2527 out:
2528         btrfs_free_path(path);
2529         return ret;
2530 }
2531
2532 static int btrfs_del_sys_chunk(struct btrfs_root *root, u64 chunk_objectid, u64
2533                         chunk_offset)
2534 {
2535         struct btrfs_super_block *super_copy = root->fs_info->super_copy;
2536         struct btrfs_disk_key *disk_key;
2537         struct btrfs_chunk *chunk;
2538         u8 *ptr;
2539         int ret = 0;
2540         u32 num_stripes;
2541         u32 array_size;
2542         u32 len = 0;
2543         u32 cur;
2544         struct btrfs_key key;
2545
2546         lock_chunks(root);
2547         array_size = btrfs_super_sys_array_size(super_copy);
2548
2549         ptr = super_copy->sys_chunk_array;
2550         cur = 0;
2551
2552         while (cur < array_size) {
2553                 disk_key = (struct btrfs_disk_key *)ptr;
2554                 btrfs_disk_key_to_cpu(&key, disk_key);
2555
2556                 len = sizeof(*disk_key);
2557
2558                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2559                         chunk = (struct btrfs_chunk *)(ptr + len);
2560                         num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2561                         len += btrfs_chunk_item_size(num_stripes);
2562                 } else {
2563                         ret = -EIO;
2564                         break;
2565                 }
2566                 if (key.objectid == chunk_objectid &&
2567                     key.offset == chunk_offset) {
2568                         memmove(ptr, ptr + len, array_size - (cur + len));
2569                         array_size -= len;
2570                         btrfs_set_super_sys_array_size(super_copy, array_size);
2571                 } else {
2572                         ptr += len;
2573                         cur += len;
2574                 }
2575         }
2576         unlock_chunks(root);
2577         return ret;
2578 }
2579
2580 int btrfs_remove_chunk(struct btrfs_trans_handle *trans,
2581                        struct btrfs_root *root, u64 chunk_offset)
2582 {
2583         struct extent_map_tree *em_tree;
2584         struct extent_map *em;
2585         struct btrfs_root *extent_root = root->fs_info->extent_root;
2586         struct map_lookup *map;
2587         u64 dev_extent_len = 0;
2588         u64 chunk_objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2589         int i, ret = 0;
2590
2591         /* Just in case */
2592         root = root->fs_info->chunk_root;
2593         em_tree = &root->fs_info->mapping_tree.map_tree;
2594
2595         read_lock(&em_tree->lock);
2596         em = lookup_extent_mapping(em_tree, chunk_offset, 1);
2597         read_unlock(&em_tree->lock);
2598
2599         if (!em || em->start > chunk_offset ||
2600             em->start + em->len < chunk_offset) {
2601                 /*
2602                  * This is a logic error, but we don't want to just rely on the
2603                  * user having built with ASSERT enabled, so if ASSERT doens't
2604                  * do anything we still error out.
2605                  */
2606                 ASSERT(0);
2607                 if (em)
2608                         free_extent_map(em);
2609                 return -EINVAL;
2610         }
2611         map = (struct map_lookup *)em->bdev;
2612
2613         for (i = 0; i < map->num_stripes; i++) {
2614                 struct btrfs_device *device = map->stripes[i].dev;
2615                 ret = btrfs_free_dev_extent(trans, device,
2616                                             map->stripes[i].physical,
2617                                             &dev_extent_len);
2618                 if (ret) {
2619                         btrfs_abort_transaction(trans, root, ret);
2620                         goto out;
2621                 }
2622
2623                 if (device->bytes_used > 0) {
2624                         lock_chunks(root);
2625                         btrfs_device_set_bytes_used(device,
2626                                         device->bytes_used - dev_extent_len);
2627                         spin_lock(&root->fs_info->free_chunk_lock);
2628                         root->fs_info->free_chunk_space += dev_extent_len;
2629                         spin_unlock(&root->fs_info->free_chunk_lock);
2630                         btrfs_clear_space_info_full(root->fs_info);
2631                         unlock_chunks(root);
2632                 }
2633
2634                 if (map->stripes[i].dev) {
2635                         ret = btrfs_update_device(trans, map->stripes[i].dev);
2636                         if (ret) {
2637                                 btrfs_abort_transaction(trans, root, ret);
2638                                 goto out;
2639                         }
2640                 }
2641         }
2642         ret = btrfs_free_chunk(trans, root, chunk_objectid, chunk_offset);
2643         if (ret) {
2644                 btrfs_abort_transaction(trans, root, ret);
2645                 goto out;
2646         }
2647
2648         trace_btrfs_chunk_free(root, map, chunk_offset, em->len);
2649
2650         if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
2651                 ret = btrfs_del_sys_chunk(root, chunk_objectid, chunk_offset);
2652                 if (ret) {
2653                         btrfs_abort_transaction(trans, root, ret);
2654                         goto out;
2655                 }
2656         }
2657
2658         ret = btrfs_remove_block_group(trans, extent_root, chunk_offset, em);
2659         if (ret) {
2660                 btrfs_abort_transaction(trans, extent_root, ret);
2661                 goto out;
2662         }
2663
2664 out:
2665         /* once for us */
2666         free_extent_map(em);
2667         return ret;
2668 }
2669
2670 static int btrfs_relocate_chunk(struct btrfs_root *root,
2671                                 u64 chunk_objectid,
2672                                 u64 chunk_offset)
2673 {
2674         struct btrfs_root *extent_root;
2675         struct btrfs_trans_handle *trans;
2676         int ret;
2677
2678         root = root->fs_info->chunk_root;
2679         extent_root = root->fs_info->extent_root;
2680
2681         ret = btrfs_can_relocate(extent_root, chunk_offset);
2682         if (ret)
2683                 return -ENOSPC;
2684
2685         /* step one, relocate all the extents inside this chunk */
2686         ret = btrfs_relocate_block_group(extent_root, chunk_offset);
2687         if (ret)
2688                 return ret;
2689
2690         trans = btrfs_start_transaction(root, 0);
2691         if (IS_ERR(trans)) {
2692                 ret = PTR_ERR(trans);
2693                 btrfs_std_error(root->fs_info, ret);
2694                 return ret;
2695         }
2696
2697         /*
2698          * step two, delete the device extents and the
2699          * chunk tree entries
2700          */
2701         ret = btrfs_remove_chunk(trans, root, chunk_offset);
2702         btrfs_end_transaction(trans, root);
2703         return ret;
2704 }
2705
2706 static int btrfs_relocate_sys_chunks(struct btrfs_root *root)
2707 {
2708         struct btrfs_root *chunk_root = root->fs_info->chunk_root;
2709         struct btrfs_path *path;
2710         struct extent_buffer *leaf;
2711         struct btrfs_chunk *chunk;
2712         struct btrfs_key key;
2713         struct btrfs_key found_key;
2714         u64 chunk_type;
2715         bool retried = false;
2716         int failed = 0;
2717         int ret;
2718
2719         path = btrfs_alloc_path();
2720         if (!path)
2721                 return -ENOMEM;
2722
2723 again:
2724         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2725         key.offset = (u64)-1;
2726         key.type = BTRFS_CHUNK_ITEM_KEY;
2727
2728         while (1) {
2729                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
2730                 if (ret < 0)
2731                         goto error;
2732                 BUG_ON(ret == 0); /* Corruption */
2733
2734                 ret = btrfs_previous_item(chunk_root, path, key.objectid,
2735                                           key.type);
2736                 if (ret < 0)
2737                         goto error;
2738                 if (ret > 0)
2739                         break;
2740
2741                 leaf = path->nodes[0];
2742                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2743
2744                 chunk = btrfs_item_ptr(leaf, path->slots[0],
2745                                        struct btrfs_chunk);
2746                 chunk_type = btrfs_chunk_type(leaf, chunk);
2747                 btrfs_release_path(path);
2748
2749                 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
2750                         ret = btrfs_relocate_chunk(chunk_root,
2751                                                    found_key.objectid,
2752                                                    found_key.offset);
2753                         if (ret == -ENOSPC)
2754                                 failed++;
2755                         else
2756                                 BUG_ON(ret);
2757                 }
2758
2759                 if (found_key.offset == 0)
2760                         break;
2761                 key.offset = found_key.offset - 1;
2762         }
2763         ret = 0;
2764         if (failed && !retried) {
2765                 failed = 0;
2766                 retried = true;
2767                 goto again;
2768         } else if (WARN_ON(failed && retried)) {
2769                 ret = -ENOSPC;
2770         }
2771 error:
2772         btrfs_free_path(path);
2773         return ret;
2774 }
2775
2776 static int insert_balance_item(struct btrfs_root *root,
2777                                struct btrfs_balance_control *bctl)
2778 {
2779         struct btrfs_trans_handle *trans;
2780         struct btrfs_balance_item *item;
2781         struct btrfs_disk_balance_args disk_bargs;
2782         struct btrfs_path *path;
2783         struct extent_buffer *leaf;
2784         struct btrfs_key key;
2785         int ret, err;
2786
2787         path = btrfs_alloc_path();
2788         if (!path)
2789                 return -ENOMEM;
2790
2791         trans = btrfs_start_transaction(root, 0);
2792         if (IS_ERR(trans)) {
2793                 btrfs_free_path(path);
2794                 return PTR_ERR(trans);
2795         }
2796
2797         key.objectid = BTRFS_BALANCE_OBJECTID;
2798         key.type = BTRFS_BALANCE_ITEM_KEY;
2799         key.offset = 0;
2800
2801         ret = btrfs_insert_empty_item(trans, root, path, &key,
2802                                       sizeof(*item));
2803         if (ret)
2804                 goto out;
2805
2806         leaf = path->nodes[0];
2807         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
2808
2809         memset_extent_buffer(leaf, 0, (unsigned long)item, sizeof(*item));
2810
2811         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
2812         btrfs_set_balance_data(leaf, item, &disk_bargs);
2813         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
2814         btrfs_set_balance_meta(leaf, item, &disk_bargs);
2815         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
2816         btrfs_set_balance_sys(leaf, item, &disk_bargs);
2817
2818         btrfs_set_balance_flags(leaf, item, bctl->flags);
2819
2820         btrfs_mark_buffer_dirty(leaf);
2821 out:
2822         btrfs_free_path(path);
2823         err = btrfs_commit_transaction(trans, root);
2824         if (err && !ret)
2825                 ret = err;
2826         return ret;
2827 }
2828
2829 static int del_balance_item(struct btrfs_root *root)
2830 {
2831         struct btrfs_trans_handle *trans;
2832         struct btrfs_path *path;
2833         struct btrfs_key key;
2834         int ret, err;
2835
2836         path = btrfs_alloc_path();
2837         if (!path)
2838                 return -ENOMEM;
2839
2840         trans = btrfs_start_transaction(root, 0);
2841         if (IS_ERR(trans)) {
2842                 btrfs_free_path(path);
2843                 return PTR_ERR(trans);
2844         }
2845
2846         key.objectid = BTRFS_BALANCE_OBJECTID;
2847         key.type = BTRFS_BALANCE_ITEM_KEY;
2848         key.offset = 0;
2849
2850         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2851         if (ret < 0)
2852                 goto out;
2853         if (ret > 0) {
2854                 ret = -ENOENT;
2855                 goto out;
2856         }
2857
2858         ret = btrfs_del_item(trans, root, path);
2859 out:
2860         btrfs_free_path(path);
2861         err = btrfs_commit_transaction(trans, root);
2862         if (err && !ret)
2863                 ret = err;
2864         return ret;
2865 }
2866
2867 /*
2868  * This is a heuristic used to reduce the number of chunks balanced on
2869  * resume after balance was interrupted.
2870  */
2871 static void update_balance_args(struct btrfs_balance_control *bctl)
2872 {
2873         /*
2874          * Turn on soft mode for chunk types that were being converted.
2875          */
2876         if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
2877                 bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
2878         if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
2879                 bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
2880         if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
2881                 bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
2882
2883         /*
2884          * Turn on usage filter if is not already used.  The idea is
2885          * that chunks that we have already balanced should be
2886          * reasonably full.  Don't do it for chunks that are being
2887          * converted - that will keep us from relocating unconverted
2888          * (albeit full) chunks.
2889          */
2890         if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
2891             !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
2892                 bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
2893                 bctl->data.usage = 90;
2894         }
2895         if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
2896             !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
2897                 bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
2898                 bctl->sys.usage = 90;
2899         }
2900         if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
2901             !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
2902                 bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
2903                 bctl->meta.usage = 90;
2904         }
2905 }
2906
2907 /*
2908  * Should be called with both balance and volume mutexes held to
2909  * serialize other volume operations (add_dev/rm_dev/resize) with
2910  * restriper.  Same goes for unset_balance_control.
2911  */
2912 static void set_balance_control(struct btrfs_balance_control *bctl)
2913 {
2914         struct btrfs_fs_info *fs_info = bctl->fs_info;
2915
2916         BUG_ON(fs_info->balance_ctl);
2917
2918         spin_lock(&fs_info->balance_lock);
2919         fs_info->balance_ctl = bctl;
2920         spin_unlock(&fs_info->balance_lock);
2921 }
2922
2923 static void unset_balance_control(struct btrfs_fs_info *fs_info)
2924 {
2925         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
2926
2927         BUG_ON(!fs_info->balance_ctl);
2928
2929         spin_lock(&fs_info->balance_lock);
2930         fs_info->balance_ctl = NULL;
2931         spin_unlock(&fs_info->balance_lock);
2932
2933         kfree(bctl);
2934 }
2935
2936 /*
2937  * Balance filters.  Return 1 if chunk should be filtered out
2938  * (should not be balanced).
2939  */
2940 static int chunk_profiles_filter(u64 chunk_type,
2941                                  struct btrfs_balance_args *bargs)
2942 {
2943         chunk_type = chunk_to_extended(chunk_type) &
2944                                 BTRFS_EXTENDED_PROFILE_MASK;
2945
2946         if (bargs->profiles & chunk_type)
2947                 return 0;
2948
2949         return 1;
2950 }
2951
2952 static int chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
2953                               struct btrfs_balance_args *bargs)
2954 {
2955         struct btrfs_block_group_cache *cache;
2956         u64 chunk_used, user_thresh;
2957         int ret = 1;
2958
2959         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2960         chunk_used = btrfs_block_group_used(&cache->item);
2961
2962         if (bargs->usage == 0)
2963                 user_thresh = 1;
2964         else if (bargs->usage > 100)
2965                 user_thresh = cache->key.offset;
2966         else
2967                 user_thresh = div_factor_fine(cache->key.offset,
2968                                               bargs->usage);
2969
2970         if (chunk_used < user_thresh)
2971                 ret = 0;
2972
2973         btrfs_put_block_group(cache);
2974         return ret;
2975 }
2976
2977 static int chunk_devid_filter(struct extent_buffer *leaf,
2978                               struct btrfs_chunk *chunk,
2979                               struct btrfs_balance_args *bargs)
2980 {
2981         struct btrfs_stripe *stripe;
2982         int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
2983         int i;
2984
2985         for (i = 0; i < num_stripes; i++) {
2986                 stripe = btrfs_stripe_nr(chunk, i);
2987                 if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
2988                         return 0;
2989         }
2990
2991         return 1;
2992 }
2993
2994 /* [pstart, pend) */
2995 static int chunk_drange_filter(struct extent_buffer *leaf,
2996                                struct btrfs_chunk *chunk,
2997                                u64 chunk_offset,
2998                                struct btrfs_balance_args *bargs)
2999 {
3000         struct btrfs_stripe *stripe;
3001         int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3002         u64 stripe_offset;
3003         u64 stripe_length;
3004         int factor;
3005         int i;
3006
3007         if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3008                 return 0;
3009
3010         if (btrfs_chunk_type(leaf, chunk) & (BTRFS_BLOCK_GROUP_DUP |
3011              BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10)) {
3012                 factor = num_stripes / 2;
3013         } else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID5) {
3014                 factor = num_stripes - 1;
3015         } else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID6) {
3016                 factor = num_stripes - 2;
3017         } else {
3018                 factor = num_stripes;
3019         }
3020
3021         for (i = 0; i < num_stripes; i++) {
3022                 stripe = btrfs_stripe_nr(chunk, i);
3023                 if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
3024                         continue;
3025
3026                 stripe_offset = btrfs_stripe_offset(leaf, stripe);
3027                 stripe_length = btrfs_chunk_length(leaf, chunk);
3028                 stripe_length = div_u64(stripe_length, factor);
3029
3030                 if (stripe_offset < bargs->pend &&
3031                     stripe_offset + stripe_length > bargs->pstart)
3032                         return 0;
3033         }
3034
3035         return 1;
3036 }
3037
3038 /* [vstart, vend) */
3039 static int chunk_vrange_filter(struct extent_buffer *leaf,
3040                                struct btrfs_chunk *chunk,
3041                                u64 chunk_offset,
3042                                struct btrfs_balance_args *bargs)
3043 {
3044         if (chunk_offset < bargs->vend &&
3045             chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
3046                 /* at least part of the chunk is inside this vrange */
3047                 return 0;
3048
3049         return 1;
3050 }
3051
3052 static int chunk_soft_convert_filter(u64 chunk_type,
3053                                      struct btrfs_balance_args *bargs)
3054 {
3055         if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3056                 return 0;
3057
3058         chunk_type = chunk_to_extended(chunk_type) &
3059                                 BTRFS_EXTENDED_PROFILE_MASK;
3060
3061         if (bargs->target == chunk_type)
3062                 return 1;
3063
3064         return 0;
3065 }
3066
3067 static int should_balance_chunk(struct btrfs_root *root,
3068                                 struct extent_buffer *leaf,
3069                                 struct btrfs_chunk *chunk, u64 chunk_offset)
3070 {
3071         struct btrfs_balance_control *bctl = root->fs_info->balance_ctl;
3072         struct btrfs_balance_args *bargs = NULL;
3073         u64 chunk_type = btrfs_chunk_type(leaf, chunk);
3074
3075         /* type filter */
3076         if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3077               (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3078                 return 0;
3079         }
3080
3081         if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3082                 bargs = &bctl->data;
3083         else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3084                 bargs = &bctl->sys;
3085         else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3086                 bargs = &bctl->meta;
3087
3088         /* profiles filter */
3089         if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3090             chunk_profiles_filter(chunk_type, bargs)) {
3091                 return 0;
3092         }
3093
3094         /* usage filter */
3095         if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3096             chunk_usage_filter(bctl->fs_info, chunk_offset, bargs)) {
3097                 return 0;
3098         }
3099
3100         /* devid filter */
3101         if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3102             chunk_devid_filter(leaf, chunk, bargs)) {
3103                 return 0;
3104         }
3105
3106         /* drange filter, makes sense only with devid filter */
3107         if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
3108             chunk_drange_filter(leaf, chunk, chunk_offset, bargs)) {
3109                 return 0;
3110         }
3111
3112         /* vrange filter */
3113         if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
3114             chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
3115                 return 0;
3116         }
3117
3118         /* soft profile changing mode */
3119         if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
3120             chunk_soft_convert_filter(chunk_type, bargs)) {
3121                 return 0;
3122         }
3123
3124         /*
3125          * limited by count, must be the last filter
3126          */
3127         if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
3128                 if (bargs->limit == 0)
3129                         return 0;
3130                 else
3131                         bargs->limit--;
3132         }
3133
3134         return 1;
3135 }
3136
3137 static int __btrfs_balance(struct btrfs_fs_info *fs_info)
3138 {
3139         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3140         struct btrfs_root *chunk_root = fs_info->chunk_root;
3141         struct btrfs_root *dev_root = fs_info->dev_root;
3142         struct list_head *devices;
3143         struct btrfs_device *device;
3144         u64 old_size;
3145         u64 size_to_free;
3146         struct btrfs_chunk *chunk;
3147         struct btrfs_path *path;
3148         struct btrfs_key key;
3149         struct btrfs_key found_key;
3150         struct btrfs_trans_handle *trans;
3151         struct extent_buffer *leaf;
3152         int slot;
3153         int ret;
3154         int enospc_errors = 0;
3155         bool counting = true;
3156         u64 limit_data = bctl->data.limit;
3157         u64 limit_meta = bctl->meta.limit;
3158         u64 limit_sys = bctl->sys.limit;
3159
3160         /* step one make some room on all the devices */
3161         devices = &fs_info->fs_devices->devices;
3162         list_for_each_entry(device, devices, dev_list) {
3163                 old_size = btrfs_device_get_total_bytes(device);
3164                 size_to_free = div_factor(old_size, 1);
3165                 size_to_free = min(size_to_free, (u64)1 * 1024 * 1024);
3166                 if (!device->writeable ||
3167                     btrfs_device_get_total_bytes(device) -
3168                     btrfs_device_get_bytes_used(device) > size_to_free ||
3169                     device->is_tgtdev_for_dev_replace)
3170                         continue;
3171
3172                 ret = btrfs_shrink_device(device, old_size - size_to_free);
3173                 if (ret == -ENOSPC)
3174                         break;
3175                 BUG_ON(ret);
3176
3177                 trans = btrfs_start_transaction(dev_root, 0);
3178                 BUG_ON(IS_ERR(trans));
3179
3180                 ret = btrfs_grow_device(trans, device, old_size);
3181                 BUG_ON(ret);
3182
3183                 btrfs_end_transaction(trans, dev_root);
3184         }
3185
3186         /* step two, relocate all the chunks */
3187         path = btrfs_alloc_path();
3188         if (!path) {
3189                 ret = -ENOMEM;
3190                 goto error;
3191         }
3192
3193         /* zero out stat counters */
3194         spin_lock(&fs_info->balance_lock);
3195         memset(&bctl->stat, 0, sizeof(bctl->stat));
3196         spin_unlock(&fs_info->balance_lock);
3197 again:
3198         if (!counting) {
3199                 bctl->data.limit = limit_data;
3200                 bctl->meta.limit = limit_meta;
3201                 bctl->sys.limit = limit_sys;
3202         }
3203         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3204         key.offset = (u64)-1;
3205         key.type = BTRFS_CHUNK_ITEM_KEY;
3206
3207         while (1) {
3208                 if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
3209                     atomic_read(&fs_info->balance_cancel_req)) {
3210                         ret = -ECANCELED;
3211                         goto error;
3212                 }
3213
3214                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3215                 if (ret < 0)
3216                         goto error;
3217
3218                 /*
3219                  * this shouldn't happen, it means the last relocate
3220                  * failed
3221                  */
3222                 if (ret == 0)
3223                         BUG(); /* FIXME break ? */
3224
3225                 ret = btrfs_previous_item(chunk_root, path, 0,
3226                                           BTRFS_CHUNK_ITEM_KEY);
3227                 if (ret) {
3228                         ret = 0;
3229                         break;
3230                 }
3231
3232                 leaf = path->nodes[0];
3233                 slot = path->slots[0];
3234                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3235
3236                 if (found_key.objectid != key.objectid)
3237                         break;
3238
3239                 chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3240
3241                 if (!counting) {
3242                         spin_lock(&fs_info->balance_lock);
3243                         bctl->stat.considered++;
3244                         spin_unlock(&fs_info->balance_lock);
3245                 }
3246
3247                 ret = should_balance_chunk(chunk_root, leaf, chunk,
3248                                            found_key.offset);
3249                 btrfs_release_path(path);
3250                 if (!ret)
3251                         goto loop;
3252
3253                 if (counting) {
3254                         spin_lock(&fs_info->balance_lock);
3255                         bctl->stat.expected++;
3256                         spin_unlock(&fs_info->balance_lock);
3257                         goto loop;
3258                 }
3259
3260                 ret = btrfs_relocate_chunk(chunk_root,
3261                                            found_key.objectid,
3262                                            found_key.offset);
3263                 if (ret && ret != -ENOSPC)
3264                         goto error;
3265                 if (ret == -ENOSPC) {
3266                         enospc_errors++;
3267                 } else {
3268                         spin_lock(&fs_info->balance_lock);
3269                         bctl->stat.completed++;
3270                         spin_unlock(&fs_info->balance_lock);
3271                 }
3272 loop:
3273                 if (found_key.offset == 0)
3274                         break;
3275                 key.offset = found_key.offset - 1;
3276         }
3277
3278         if (counting) {
3279                 btrfs_release_path(path);
3280                 counting = false;
3281                 goto again;
3282         }
3283 error:
3284         btrfs_free_path(path);
3285         if (enospc_errors) {
3286                 btrfs_info(fs_info, "%d enospc errors during balance",
3287                        enospc_errors);
3288                 if (!ret)
3289                         ret = -ENOSPC;
3290         }
3291
3292         return ret;
3293 }
3294
3295 /**
3296  * alloc_profile_is_valid - see if a given profile is valid and reduced
3297  * @flags: profile to validate
3298  * @extended: if true @flags is treated as an extended profile
3299  */
3300 static int alloc_profile_is_valid(u64 flags, int extended)
3301 {
3302         u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
3303                                BTRFS_BLOCK_GROUP_PROFILE_MASK);
3304
3305         flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
3306
3307         /* 1) check that all other bits are zeroed */
3308         if (flags & ~mask)
3309                 return 0;
3310
3311         /* 2) see if profile is reduced */
3312         if (flags == 0)
3313                 return !extended; /* "0" is valid for usual profiles */
3314
3315         /* true if exactly one bit set */
3316         return (flags & (flags - 1)) == 0;
3317 }
3318
3319 static inline int balance_need_close(struct btrfs_fs_info *fs_info)
3320 {
3321         /* cancel requested || normal exit path */
3322         return atomic_read(&fs_info->balance_cancel_req) ||
3323                 (atomic_read(&fs_info->balance_pause_req) == 0 &&
3324                  atomic_read(&fs_info->balance_cancel_req) == 0);
3325 }
3326
3327 static void __cancel_balance(struct btrfs_fs_info *fs_info)
3328 {
3329         int ret;
3330
3331         unset_balance_control(fs_info);
3332         ret = del_balance_item(fs_info->tree_root);
3333         if (ret)
3334                 btrfs_std_error(fs_info, ret);
3335
3336         atomic_set(&fs_info->mutually_exclusive_operation_running, 0);
3337 }
3338
3339 /*
3340  * Should be called with both balance and volume mutexes held
3341  */
3342 int btrfs_balance(struct btrfs_balance_control *bctl,
3343                   struct btrfs_ioctl_balance_args *bargs)
3344 {
3345         struct btrfs_fs_info *fs_info = bctl->fs_info;
3346         u64 allowed;
3347         int mixed = 0;
3348         int ret;
3349         u64 num_devices;
3350         unsigned seq;
3351
3352         if (btrfs_fs_closing(fs_info) ||
3353             atomic_read(&fs_info->balance_pause_req) ||
3354             atomic_read(&fs_info->balance_cancel_req)) {
3355                 ret = -EINVAL;
3356                 goto out;
3357         }
3358
3359         allowed = btrfs_super_incompat_flags(fs_info->super_copy);
3360         if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
3361                 mixed = 1;
3362
3363         /*
3364          * In case of mixed groups both data and meta should be picked,
3365          * and identical options should be given for both of them.
3366          */
3367         allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
3368         if (mixed && (bctl->flags & allowed)) {
3369                 if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
3370                     !(bctl->flags & BTRFS_BALANCE_METADATA) ||
3371                     memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
3372                         btrfs_err(fs_info, "with mixed groups data and "
3373                                    "metadata balance options must be the same");
3374                         ret = -EINVAL;
3375                         goto out;
3376                 }
3377         }
3378
3379         num_devices = fs_info->fs_devices->num_devices;
3380         btrfs_dev_replace_lock(&fs_info->dev_replace);
3381         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
3382                 BUG_ON(num_devices < 1);
3383                 num_devices--;
3384         }
3385         btrfs_dev_replace_unlock(&fs_info->dev_replace);
3386         allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
3387         if (num_devices == 1)
3388                 allowed |= BTRFS_BLOCK_GROUP_DUP;
3389         else if (num_devices > 1)
3390                 allowed |= (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID1);
3391         if (num_devices > 2)
3392                 allowed |= BTRFS_BLOCK_GROUP_RAID5;
3393         if (num_devices > 3)
3394                 allowed |= (BTRFS_BLOCK_GROUP_RAID10 |
3395                             BTRFS_BLOCK_GROUP_RAID6);
3396         if ((bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3397             (!alloc_profile_is_valid(bctl->data.target, 1) ||
3398              (bctl->data.target & ~allowed))) {
3399                 btrfs_err(fs_info, "unable to start balance with target "
3400                            "data profile %llu",
3401                        bctl->data.target);
3402                 ret = -EINVAL;
3403                 goto out;
3404         }
3405         if ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3406             (!alloc_profile_is_valid(bctl->meta.target, 1) ||
3407              (bctl->meta.target & ~allowed))) {
3408                 btrfs_err(fs_info,
3409                            "unable to start balance with target metadata profile %llu",
3410                        bctl->meta.target);
3411                 ret = -EINVAL;
3412                 goto out;
3413         }
3414         if ((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3415             (!alloc_profile_is_valid(bctl->sys.target, 1) ||
3416              (bctl->sys.target & ~allowed))) {
3417                 btrfs_err(fs_info,
3418                            "unable to start balance with target system profile %llu",
3419                        bctl->sys.target);
3420                 ret = -EINVAL;
3421                 goto out;
3422         }
3423
3424         /* allow dup'ed data chunks only in mixed mode */
3425         if (!mixed && (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3426             (bctl->data.target & BTRFS_BLOCK_GROUP_DUP)) {
3427                 btrfs_err(fs_info, "dup for data is not allowed");
3428                 ret = -EINVAL;
3429                 goto out;
3430         }
3431
3432         /* allow to reduce meta or sys integrity only if force set */
3433         allowed = BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
3434                         BTRFS_BLOCK_GROUP_RAID10 |
3435                         BTRFS_BLOCK_GROUP_RAID5 |
3436                         BTRFS_BLOCK_GROUP_RAID6;
3437         do {
3438                 seq = read_seqbegin(&fs_info->profiles_lock);
3439
3440                 if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3441                      (fs_info->avail_system_alloc_bits & allowed) &&
3442                      !(bctl->sys.target & allowed)) ||
3443                     ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
3444                      (fs_info->avail_metadata_alloc_bits & allowed) &&
3445                      !(bctl->meta.target & allowed))) {
3446                         if (bctl->flags & BTRFS_BALANCE_FORCE) {
3447                                 btrfs_info(fs_info, "force reducing metadata integrity");
3448                         } else {
3449                                 btrfs_err(fs_info, "balance will reduce metadata "
3450                                            "integrity, use force if you want this");
3451                                 ret = -EINVAL;
3452                                 goto out;
3453                         }
3454                 }
3455         } while (read_seqretry(&fs_info->profiles_lock, seq));
3456
3457         if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3458                 int num_tolerated_disk_barrier_failures;
3459                 u64 target = bctl->sys.target;
3460
3461                 num_tolerated_disk_barrier_failures =
3462                         btrfs_calc_num_tolerated_disk_barrier_failures(fs_info);
3463                 if (num_tolerated_disk_barrier_failures > 0 &&
3464                     (target &
3465                      (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID0 |
3466                       BTRFS_AVAIL_ALLOC_BIT_SINGLE)))
3467                         num_tolerated_disk_barrier_failures = 0;
3468                 else if (num_tolerated_disk_barrier_failures > 1 &&
3469                          (target &
3470                           (BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10)))
3471                         num_tolerated_disk_barrier_failures = 1;
3472
3473                 fs_info->num_tolerated_disk_barrier_failures =
3474                         num_tolerated_disk_barrier_failures;
3475         }
3476
3477         ret = insert_balance_item(fs_info->tree_root, bctl);
3478         if (ret && ret != -EEXIST)
3479                 goto out;
3480
3481         if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
3482                 BUG_ON(ret == -EEXIST);
3483                 set_balance_control(bctl);
3484         } else {
3485                 BUG_ON(ret != -EEXIST);
3486                 spin_lock(&fs_info->balance_lock);
3487                 update_balance_args(bctl);
3488                 spin_unlock(&fs_info->balance_lock);
3489         }
3490
3491         atomic_inc(&fs_info->balance_running);
3492         mutex_unlock(&fs_info->balance_mutex);
3493
3494         ret = __btrfs_balance(fs_info);
3495
3496         mutex_lock(&fs_info->balance_mutex);
3497         atomic_dec(&fs_info->balance_running);
3498
3499         if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3500                 fs_info->num_tolerated_disk_barrier_failures =
3501                         btrfs_calc_num_tolerated_disk_barrier_failures(fs_info);
3502         }
3503
3504         if (bargs) {
3505                 memset(bargs, 0, sizeof(*bargs));
3506                 update_ioctl_balance_args(fs_info, 0, bargs);
3507         }
3508
3509         if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
3510             balance_need_close(fs_info)) {
3511                 __cancel_balance(fs_info);
3512         }
3513
3514         wake_up(&fs_info->balance_wait_q);
3515
3516         return ret;
3517 out:
3518         if (bctl->flags & BTRFS_BALANCE_RESUME)
3519                 __cancel_balance(fs_info);
3520         else {
3521                 kfree(bctl);
3522                 atomic_set(&fs_info->mutually_exclusive_operation_running, 0);
3523         }
3524         return ret;
3525 }
3526
3527 static int balance_kthread(void *data)
3528 {
3529         struct btrfs_fs_info *fs_info = data;
3530         int ret = 0;
3531
3532         mutex_lock(&fs_info->volume_mutex);
3533         mutex_lock(&fs_info->balance_mutex);
3534
3535         if (fs_info->balance_ctl) {
3536                 btrfs_info(fs_info, "continuing balance");
3537                 ret = btrfs_balance(fs_info->balance_ctl, NULL);
3538         }
3539
3540         mutex_unlock(&fs_info->balance_mutex);
3541         mutex_unlock(&fs_info->volume_mutex);
3542
3543         return ret;
3544 }
3545
3546 int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
3547 {
3548         struct task_struct *tsk;
3549
3550         spin_lock(&fs_info->balance_lock);
3551         if (!fs_info->balance_ctl) {
3552                 spin_unlock(&fs_info->balance_lock);
3553                 return 0;
3554         }
3555         spin_unlock(&fs_info->balance_lock);
3556
3557         if (btrfs_test_opt(fs_info->tree_root, SKIP_BALANCE)) {
3558                 btrfs_info(fs_info, "force skipping balance");
3559                 return 0;
3560         }
3561
3562         tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
3563         return PTR_ERR_OR_ZERO(tsk);
3564 }
3565
3566 int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
3567 {
3568         struct btrfs_balance_control *bctl;
3569         struct btrfs_balance_item *item;
3570         struct btrfs_disk_balance_args disk_bargs;
3571         struct btrfs_path *path;
3572         struct extent_buffer *leaf;
3573         struct btrfs_key key;
3574         int ret;
3575
3576         path = btrfs_alloc_path();
3577         if (!path)
3578                 return -ENOMEM;
3579
3580         key.objectid = BTRFS_BALANCE_OBJECTID;
3581         key.type = BTRFS_BALANCE_ITEM_KEY;
3582         key.offset = 0;
3583
3584         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
3585         if (ret < 0)
3586                 goto out;
3587         if (ret > 0) { /* ret = -ENOENT; */
3588                 ret = 0;
3589                 goto out;
3590         }
3591
3592         bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
3593         if (!bctl) {
3594                 ret = -ENOMEM;
3595                 goto out;
3596         }
3597
3598         leaf = path->nodes[0];
3599         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3600
3601         bctl->fs_info = fs_info;
3602         bctl->flags = btrfs_balance_flags(leaf, item);
3603         bctl->flags |= BTRFS_BALANCE_RESUME;
3604
3605         btrfs_balance_data(leaf, item, &disk_bargs);
3606         btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
3607         btrfs_balance_meta(leaf, item, &disk_bargs);
3608         btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
3609         btrfs_balance_sys(leaf, item, &disk_bargs);
3610         btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);
3611
3612         WARN_ON(atomic_xchg(&fs_info->mutually_exclusive_operation_running, 1));
3613
3614         mutex_lock(&fs_info->volume_mutex);
3615         mutex_lock(&fs_info->balance_mutex);
3616
3617         set_balance_control(bctl);
3618
3619         mutex_unlock(&fs_info->balance_mutex);
3620         mutex_unlock(&fs_info->volume_mutex);
3621 out:
3622         btrfs_free_path(path);
3623         return ret;
3624 }
3625
3626 int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
3627 {
3628         int ret = 0;
3629
3630         mutex_lock(&fs_info->balance_mutex);
3631         if (!fs_info->balance_ctl) {
3632                 mutex_unlock(&fs_info->balance_mutex);
3633                 return -ENOTCONN;
3634         }
3635
3636         if (atomic_read(&fs_info->balance_running)) {
3637                 atomic_inc(&fs_info->balance_pause_req);
3638                 mutex_unlock(&fs_info->balance_mutex);
3639
3640                 wait_event(fs_info->balance_wait_q,
3641                            atomic_read(&fs_info->balance_running) == 0);
3642
3643                 mutex_lock(&fs_info->balance_mutex);
3644                 /* we are good with balance_ctl ripped off from under us */
3645                 BUG_ON(atomic_read(&fs_info->balance_running));
3646                 atomic_dec(&fs_info->balance_pause_req);
3647         } else {
3648                 ret = -ENOTCONN;
3649         }
3650
3651         mutex_unlock(&fs_info->balance_mutex);
3652         return ret;
3653 }
3654
3655 int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
3656 {
3657         if (fs_info->sb->s_flags & MS_RDONLY)
3658                 return -EROFS;
3659
3660         mutex_lock(&fs_info->balance_mutex);
3661         if (!fs_info->balance_ctl) {
3662                 mutex_unlock(&fs_info->balance_mutex);
3663                 return -ENOTCONN;
3664         }
3665
3666         atomic_inc(&fs_info->balance_cancel_req);
3667         /*
3668          * if we are running just wait and return, balance item is
3669          * deleted in btrfs_balance in this case
3670          */
3671         if (atomic_read(&fs_info->balance_running)) {
3672                 mutex_unlock(&fs_info->balance_mutex);
3673                 wait_event(fs_info->balance_wait_q,
3674                            atomic_read(&fs_info->balance_running) == 0);
3675                 mutex_lock(&fs_info->balance_mutex);
3676         } else {
3677                 /* __cancel_balance needs volume_mutex */
3678                 mutex_unlock(&fs_info->balance_mutex);
3679                 mutex_lock(&fs_info->volume_mutex);
3680                 mutex_lock(&fs_info->balance_mutex);
3681
3682                 if (fs_info->balance_ctl)
3683                         __cancel_balance(fs_info);
3684
3685                 mutex_unlock(&fs_info->volume_mutex);
3686         }
3687
3688         BUG_ON(fs_info->balance_ctl || atomic_read(&fs_info->balance_running));
3689         atomic_dec(&fs_info->balance_cancel_req);
3690         mutex_unlock(&fs_info->balance_mutex);
3691         return 0;
3692 }
3693
3694 static int btrfs_uuid_scan_kthread(void *data)
3695 {
3696         struct btrfs_fs_info *fs_info = data;
3697         struct btrfs_root *root = fs_info->tree_root;
3698         struct btrfs_key key;
3699         struct btrfs_key max_key;
3700         struct btrfs_path *path = NULL;
3701         int ret = 0;
3702         struct extent_buffer *eb;
3703         int slot;
3704         struct btrfs_root_item root_item;
3705         u32 item_size;
3706         struct btrfs_trans_handle *trans = NULL;
3707
3708         path = btrfs_alloc_path();
3709         if (!path) {
3710                 ret = -ENOMEM;
3711                 goto out;
3712         }
3713
3714         key.objectid = 0;
3715         key.type = BTRFS_ROOT_ITEM_KEY;
3716         key.offset = 0;
3717
3718         max_key.objectid = (u64)-1;
3719         max_key.type = BTRFS_ROOT_ITEM_KEY;
3720         max_key.offset = (u64)-1;
3721
3722         while (1) {
3723                 ret = btrfs_search_forward(root, &key, path, 0);
3724                 if (ret) {
3725                         if (ret > 0)
3726                                 ret = 0;
3727                         break;
3728                 }
3729
3730                 if (key.type != BTRFS_ROOT_ITEM_KEY ||
3731                     (key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
3732                      key.objectid != BTRFS_FS_TREE_OBJECTID) ||
3733                     key.objectid > BTRFS_LAST_FREE_OBJECTID)
3734                         goto skip;
3735
3736                 eb = path->nodes[0];
3737                 slot = path->slots[0];
3738                 item_size = btrfs_item_size_nr(eb, slot);
3739                 if (item_size < sizeof(root_item))
3740                         goto skip;
3741
3742                 read_extent_buffer(eb, &root_item,
3743                                    btrfs_item_ptr_offset(eb, slot),
3744                                    (int)sizeof(root_item));
3745                 if (btrfs_root_refs(&root_item) == 0)
3746                         goto skip;
3747
3748                 if (!btrfs_is_empty_uuid(root_item.uuid) ||
3749                     !btrfs_is_empty_uuid(root_item.received_uuid)) {
3750                         if (trans)
3751                                 goto update_tree;
3752
3753                         btrfs_release_path(path);
3754                         /*
3755                          * 1 - subvol uuid item
3756                          * 1 - received_subvol uuid item
3757                          */
3758                         trans = btrfs_start_transaction(fs_info->uuid_root, 2);
3759                         if (IS_ERR(trans)) {
3760                                 ret = PTR_ERR(trans);
3761                                 break;
3762                         }
3763                         continue;
3764                 } else {
3765                         goto skip;
3766                 }
3767 update_tree:
3768                 if (!btrfs_is_empty_uuid(root_item.uuid)) {
3769                         ret = btrfs_uuid_tree_add(trans, fs_info->uuid_root,
3770                                                   root_item.uuid,
3771                                                   BTRFS_UUID_KEY_SUBVOL,
3772                                                   key.objectid);
3773                         if (ret < 0) {
3774                                 btrfs_warn(fs_info, "uuid_tree_add failed %d",
3775                                         ret);
3776                                 break;
3777                         }
3778                 }
3779
3780                 if (!btrfs_is_empty_uuid(root_item.received_uuid)) {
3781                         ret = btrfs_uuid_tree_add(trans, fs_info->uuid_root,
3782                                                   root_item.received_uuid,
3783                                                  BTRFS_UUID_KEY_RECEIVED_SUBVOL,
3784                                                   key.objectid);
3785                         if (ret < 0) {
3786                                 btrfs_warn(fs_info, "uuid_tree_add failed %d",
3787                                         ret);
3788                                 break;
3789                         }
3790                 }
3791
3792 skip:
3793                 if (trans) {
3794                         ret = btrfs_end_transaction(trans, fs_info->uuid_root);
3795                         trans = NULL;
3796                         if (ret)
3797                                 break;
3798                 }
3799
3800                 btrfs_release_path(path);
3801                 if (key.offset < (u64)-1) {
3802                         key.offset++;
3803                 } else if (key.type < BTRFS_ROOT_ITEM_KEY) {
3804                         key.offset = 0;
3805                         key.type = BTRFS_ROOT_ITEM_KEY;
3806                 } else if (key.objectid < (u64)-1) {
3807                         key.offset = 0;
3808                         key.type = BTRFS_ROOT_ITEM_KEY;
3809                         key.objectid++;
3810                 } else {
3811                         break;
3812                 }
3813                 cond_resched();
3814         }
3815
3816 out:
3817         btrfs_free_path(path);
3818         if (trans && !IS_ERR(trans))
3819                 btrfs_end_transaction(trans, fs_info->uuid_root);
3820         if (ret)
3821                 btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
3822         else
3823                 fs_info->update_uuid_tree_gen = 1;
3824         up(&fs_info->uuid_tree_rescan_sem);
3825         return 0;
3826 }
3827
3828 /*
3829  * Callback for btrfs_uuid_tree_iterate().
3830  * returns:
3831  * 0    check succeeded, the entry is not outdated.
3832  * < 0  if an error occured.
3833  * > 0  if the check failed, which means the caller shall remove the entry.
3834  */
3835 static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info,
3836                                        u8 *uuid, u8 type, u64 subid)
3837 {
3838         struct btrfs_key key;
3839         int ret = 0;
3840         struct btrfs_root *subvol_root;
3841
3842         if (type != BTRFS_UUID_KEY_SUBVOL &&
3843             type != BTRFS_UUID_KEY_RECEIVED_SUBVOL)
3844                 goto out;
3845
3846         key.objectid = subid;
3847         key.type = BTRFS_ROOT_ITEM_KEY;
3848         key.offset = (u64)-1;
3849         subvol_root = btrfs_read_fs_root_no_name(fs_info, &key);
3850         if (IS_ERR(subvol_root)) {
3851                 ret = PTR_ERR(subvol_root);
3852                 if (ret == -ENOENT)
3853                         ret = 1;
3854                 goto out;
3855         }
3856
3857         switch (type) {
3858         case BTRFS_UUID_KEY_SUBVOL:
3859                 if (memcmp(uuid, subvol_root->root_item.uuid, BTRFS_UUID_SIZE))
3860                         ret = 1;
3861                 break;
3862         case BTRFS_UUID_KEY_RECEIVED_SUBVOL:
3863                 if (memcmp(uuid, subvol_root->root_item.received_uuid,
3864                            BTRFS_UUID_SIZE))
3865                         ret = 1;
3866                 break;
3867         }
3868
3869 out:
3870         return ret;
3871 }
3872
3873 static int btrfs_uuid_rescan_kthread(void *data)
3874 {
3875         struct btrfs_fs_info *fs_info = (struct btrfs_fs_info *)data;
3876         int ret;
3877
3878         /*
3879          * 1st step is to iterate through the existing UUID tree and
3880          * to delete all entries that contain outdated data.
3881          * 2nd step is to add all missing entries to the UUID tree.
3882          */
3883         ret = btrfs_uuid_tree_iterate(fs_info, btrfs_check_uuid_tree_entry);
3884         if (ret < 0) {
3885                 btrfs_warn(fs_info, "iterating uuid_tree failed %d", ret);
3886                 up(&fs_info->uuid_tree_rescan_sem);
3887                 return ret;
3888         }
3889         return btrfs_uuid_scan_kthread(data);
3890 }
3891
3892 int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
3893 {
3894         struct btrfs_trans_handle *trans;
3895         struct btrfs_root *tree_root = fs_info->tree_root;
3896         struct btrfs_root *uuid_root;
3897         struct task_struct *task;
3898         int ret;
3899
3900         /*
3901          * 1 - root node
3902          * 1 - root item
3903          */
3904         trans = btrfs_start_transaction(tree_root, 2);
3905         if (IS_ERR(trans))
3906                 return PTR_ERR(trans);
3907
3908         uuid_root = btrfs_create_tree(trans, fs_info,
3909                                       BTRFS_UUID_TREE_OBJECTID);
3910         if (IS_ERR(uuid_root)) {
3911                 btrfs_abort_transaction(trans, tree_root,
3912                                         PTR_ERR(uuid_root));
3913                 return PTR_ERR(uuid_root);
3914         }
3915
3916         fs_info->uuid_root = uuid_root;
3917
3918         ret = btrfs_commit_transaction(trans, tree_root);
3919         if (ret)
3920                 return ret;
3921
3922         down(&fs_info->uuid_tree_rescan_sem);
3923         task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
3924         if (IS_ERR(task)) {
3925                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3926                 btrfs_warn(fs_info, "failed to start uuid_scan task");
3927                 up(&fs_info->uuid_tree_rescan_sem);
3928                 return PTR_ERR(task);
3929         }
3930
3931         return 0;
3932 }
3933
3934 int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
3935 {
3936         struct task_struct *task;
3937
3938         down(&fs_info->uuid_tree_rescan_sem);
3939         task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
3940         if (IS_ERR(task)) {
3941                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3942                 btrfs_warn(fs_info, "failed to start uuid_rescan task");
3943                 up(&fs_info->uuid_tree_rescan_sem);
3944                 return PTR_ERR(task);
3945         }
3946
3947         return 0;
3948 }
3949
3950 /*
3951  * shrinking a device means finding all of the device extents past
3952  * the new size, and then following the back refs to the chunks.
3953  * The chunk relocation code actually frees the device extent
3954  */
3955 int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
3956 {
3957         struct btrfs_trans_handle *trans;
3958         struct btrfs_root *root = device->dev_root;
3959         struct btrfs_dev_extent *dev_extent = NULL;
3960         struct btrfs_path *path;
3961         u64 length;
3962         u64 chunk_objectid;
3963         u64 chunk_offset;
3964         int ret;
3965         int slot;
3966         int failed = 0;
3967         bool retried = false;
3968         struct extent_buffer *l;
3969         struct btrfs_key key;
3970         struct btrfs_super_block *super_copy = root->fs_info->super_copy;
3971         u64 old_total = btrfs_super_total_bytes(super_copy);
3972         u64 old_size = btrfs_device_get_total_bytes(device);
3973         u64 diff = old_size - new_size;
3974
3975         if (device->is_tgtdev_for_dev_replace)
3976                 return -EINVAL;
3977
3978         path = btrfs_alloc_path();
3979         if (!path)
3980                 return -ENOMEM;
3981
3982         path->reada = 2;
3983
3984         lock_chunks(root);
3985
3986         btrfs_device_set_total_bytes(device, new_size);
3987         if (device->writeable) {
3988                 device->fs_devices->total_rw_bytes -= diff;
3989                 spin_lock(&root->fs_info->free_chunk_lock);
3990                 root->fs_info->free_chunk_space -= diff;
3991                 spin_unlock(&root->fs_info->free_chunk_lock);
3992         }
3993         unlock_chunks(root);
3994
3995 again:
3996         key.objectid = device->devid;
3997         key.offset = (u64)-1;
3998         key.type = BTRFS_DEV_EXTENT_KEY;
3999
4000         do {
4001                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4002                 if (ret < 0)
4003                         goto done;
4004
4005                 ret = btrfs_previous_item(root, path, 0, key.type);
4006                 if (ret < 0)
4007                         goto done;
4008                 if (ret) {
4009                         ret = 0;
4010                         btrfs_release_path(path);
4011                         break;
4012                 }
4013
4014                 l = path->nodes[0];
4015                 slot = path->slots[0];
4016                 btrfs_item_key_to_cpu(l, &key, path->slots[0]);
4017
4018                 if (key.objectid != device->devid) {
4019                         btrfs_release_path(path);
4020                         break;
4021                 }
4022
4023                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
4024                 length = btrfs_dev_extent_length(l, dev_extent);
4025
4026                 if (key.offset + length <= new_size) {
4027                         btrfs_release_path(path);
4028                         break;
4029                 }
4030
4031                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
4032                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
4033                 btrfs_release_path(path);
4034
4035                 ret = btrfs_relocate_chunk(root, chunk_objectid, chunk_offset);
4036                 if (ret && ret != -ENOSPC)
4037                         goto done;
4038                 if (ret == -ENOSPC)
4039                         failed++;
4040         } while (key.offset-- > 0);
4041
4042         if (failed && !retried) {
4043                 failed = 0;
4044                 retried = true;
4045                 goto again;
4046         } else if (failed && retried) {
4047                 ret = -ENOSPC;
4048                 lock_chunks(root);
4049
4050                 btrfs_device_set_total_bytes(device, old_size);
4051                 if (device->writeable)
4052                         device->fs_devices->total_rw_bytes += diff;
4053                 spin_lock(&root->fs_info->free_chunk_lock);
4054                 root->fs_info->free_chunk_space += diff;
4055                 spin_unlock(&root->fs_info->free_chunk_lock);
4056                 unlock_chunks(root);
4057                 goto done;
4058         }
4059
4060         /* Shrinking succeeded, else we would be at "done". */
4061         trans = btrfs_start_transaction(root, 0);
4062         if (IS_ERR(trans)) {
4063                 ret = PTR_ERR(trans);
4064                 goto done;
4065         }
4066
4067         lock_chunks(root);
4068         btrfs_device_set_disk_total_bytes(device, new_size);
4069         if (list_empty(&device->resized_list))
4070                 list_add_tail(&device->resized_list,
4071                               &root->fs_info->fs_devices->resized_devices);
4072
4073         WARN_ON(diff > old_total);
4074         btrfs_set_super_total_bytes(super_copy, old_total - diff);
4075         unlock_chunks(root);
4076
4077         /* Now btrfs_update_device() will change the on-disk size. */
4078         ret = btrfs_update_device(trans, device);
4079         btrfs_end_transaction(trans, root);
4080 done:
4081         btrfs_free_path(path);
4082         return ret;
4083 }
4084
4085 static int btrfs_add_system_chunk(struct btrfs_root *root,
4086                            struct btrfs_key *key,
4087                            struct btrfs_chunk *chunk, int item_size)
4088 {
4089         struct btrfs_super_block *super_copy = root->fs_info->super_copy;
4090         struct btrfs_disk_key disk_key;
4091         u32 array_size;
4092         u8 *ptr;
4093
4094         lock_chunks(root);
4095         array_size = btrfs_super_sys_array_size(super_copy);
4096         if (array_size + item_size + sizeof(disk_key)
4097                         > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
4098                 unlock_chunks(root);
4099                 return -EFBIG;
4100         }
4101
4102         ptr = super_copy->sys_chunk_array + array_size;
4103         btrfs_cpu_key_to_disk(&disk_key, key);
4104         memcpy(ptr, &disk_key, sizeof(disk_key));
4105         ptr += sizeof(disk_key);
4106         memcpy(ptr, chunk, item_size);
4107         item_size += sizeof(disk_key);
4108         btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
4109         unlock_chunks(root);
4110
4111         return 0;
4112 }
4113
4114 /*
4115  * sort the devices in descending order by max_avail, total_avail
4116  */
4117 static int btrfs_cmp_device_info(const void *a, const void *b)
4118 {
4119         const struct btrfs_device_info *di_a = a;
4120         const struct btrfs_device_info *di_b = b;
4121
4122         if (di_a->max_avail > di_b->max_avail)
4123                 return -1;
4124         if (di_a->max_avail < di_b->max_avail)
4125                 return 1;
4126         if (di_a->total_avail > di_b->total_avail)
4127                 return -1;
4128         if (di_a->total_avail < di_b->total_avail)
4129                 return 1;
4130         return 0;
4131 }
4132
4133 static const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
4134         [BTRFS_RAID_RAID10] = {
4135                 .sub_stripes    = 2,
4136                 .dev_stripes    = 1,
4137                 .devs_max       = 0,    /* 0 == as many as possible */
4138                 .devs_min       = 4,
4139                 .devs_increment = 2,
4140                 .ncopies        = 2,
4141         },
4142         [BTRFS_RAID_RAID1] = {
4143                 .sub_stripes    = 1,
4144                 .dev_stripes    = 1,
4145                 .devs_max       = 2,
4146                 .devs_min       = 2,
4147                 .devs_increment = 2,
4148                 .ncopies        = 2,
4149         },
4150         [BTRFS_RAID_DUP] = {
4151                 .sub_stripes    = 1,
4152                 .dev_stripes    = 2,
4153                 .devs_max       = 1,
4154                 .devs_min       = 1,
4155                 .devs_increment = 1,
4156                 .ncopies        = 2,
4157         },
4158         [BTRFS_RAID_RAID0] = {
4159                 .sub_stripes    = 1,
4160                 .dev_stripes    = 1,
4161                 .devs_max       = 0,
4162                 .devs_min       = 2,
4163                 .devs_increment = 1,
4164                 .ncopies        = 1,
4165         },
4166         [BTRFS_RAID_SINGLE] = {
4167                 .sub_stripes    = 1,
4168                 .dev_stripes    = 1,
4169                 .devs_max       = 1,
4170                 .devs_min       = 1,
4171                 .devs_increment = 1,
4172                 .ncopies        = 1,
4173         },
4174         [BTRFS_RAID_RAID5] = {
4175                 .sub_stripes    = 1,
4176                 .dev_stripes    = 1,
4177                 .devs_max       = 0,
4178                 .devs_min       = 2,
4179                 .devs_increment = 1,
4180                 .ncopies        = 2,
4181         },
4182         [BTRFS_RAID_RAID6] = {
4183                 .sub_stripes    = 1,
4184                 .dev_stripes    = 1,
4185                 .devs_max       = 0,
4186                 .devs_min       = 3,
4187                 .devs_increment = 1,
4188                 .ncopies        = 3,
4189         },
4190 };
4191
4192 static u32 find_raid56_stripe_len(u32 data_devices, u32 dev_stripe_target)
4193 {
4194         /* TODO allow them to set a preferred stripe size */
4195         return 64 * 1024;
4196 }
4197
4198 static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
4199 {
4200         if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
4201                 return;
4202
4203         btrfs_set_fs_incompat(info, RAID56);
4204 }
4205
4206 #define BTRFS_MAX_DEVS(r) ((BTRFS_LEAF_DATA_SIZE(r)             \
4207                         - sizeof(struct btrfs_item)             \
4208                         - sizeof(struct btrfs_chunk))           \
4209                         / sizeof(struct btrfs_stripe) + 1)
4210
4211 #define BTRFS_MAX_DEVS_SYS_CHUNK ((BTRFS_SYSTEM_CHUNK_ARRAY_SIZE        \
4212                                 - 2 * sizeof(struct btrfs_disk_key)     \
4213                                 - 2 * sizeof(struct btrfs_chunk))       \
4214                                 / sizeof(struct btrfs_stripe) + 1)
4215
4216 static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
4217                                struct btrfs_root *extent_root, u64 start,
4218                                u64 type)
4219 {
4220         struct btrfs_fs_info *info = extent_root->fs_info;
4221         struct btrfs_fs_devices *fs_devices = info->fs_devices;
4222         struct list_head *cur;
4223         struct map_lookup *map = NULL;
4224         struct extent_map_tree *em_tree;
4225         struct extent_map *em;
4226         struct btrfs_device_info *devices_info = NULL;
4227         u64 total_avail;
4228         int num_stripes;        /* total number of stripes to allocate */
4229         int data_stripes;       /* number of stripes that count for
4230                                    block group size */
4231         int sub_stripes;        /* sub_stripes info for map */
4232         int dev_stripes;        /* stripes per dev */
4233         int devs_max;           /* max devs to use */
4234         int devs_min;           /* min devs needed */
4235         int devs_increment;     /* ndevs has to be a multiple of this */
4236         int ncopies;            /* how many copies to data has */
4237         int ret;
4238         u64 max_stripe_size;
4239         u64 max_chunk_size;
4240         u64 stripe_size;
4241         u64 num_bytes;
4242         u64 raid_stripe_len = BTRFS_STRIPE_LEN;
4243         int ndevs;
4244         int i;
4245         int j;
4246         int index;
4247
4248         BUG_ON(!alloc_profile_is_valid(type, 0));
4249
4250         if (list_empty(&fs_devices->alloc_list))
4251                 return -ENOSPC;
4252
4253         index = __get_raid_index(type);
4254
4255         sub_stripes = btrfs_raid_array[index].sub_stripes;
4256         dev_stripes = btrfs_raid_array[index].dev_stripes;
4257         devs_max = btrfs_raid_array[index].devs_max;
4258         devs_min = btrfs_raid_array[index].devs_min;
4259         devs_increment = btrfs_raid_array[index].devs_increment;
4260         ncopies = btrfs_raid_array[index].ncopies;
4261
4262         if (type & BTRFS_BLOCK_GROUP_DATA) {
4263                 max_stripe_size = 1024 * 1024 * 1024;
4264                 max_chunk_size = 10 * max_stripe_size;
4265                 if (!devs_max)
4266                         devs_max = BTRFS_MAX_DEVS(info->chunk_root);
4267         } else if (type & BTRFS_BLOCK_GROUP_METADATA) {
4268                 /* for larger filesystems, use larger metadata chunks */
4269                 if (fs_devices->total_rw_bytes > 50ULL * 1024 * 1024 * 1024)
4270                         max_stripe_size = 1024 * 1024 * 1024;
4271                 else
4272                         max_stripe_size = 256 * 1024 * 1024;
4273                 max_chunk_size = max_stripe_size;
4274                 if (!devs_max)
4275                         devs_max = BTRFS_MAX_DEVS(info->chunk_root);
4276         } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
4277                 max_stripe_size = 32 * 1024 * 1024;
4278                 max_chunk_size = 2 * max_stripe_size;
4279                 if (!devs_max)
4280                         devs_max = BTRFS_MAX_DEVS_SYS_CHUNK;
4281         } else {
4282                 btrfs_err(info, "invalid chunk type 0x%llx requested",
4283                        type);
4284                 BUG_ON(1);
4285         }
4286
4287         /* we don't want a chunk larger than 10% of writeable space */
4288         max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
4289                              max_chunk_size);
4290
4291         devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
4292                                GFP_NOFS);
4293         if (!devices_info)
4294                 return -ENOMEM;
4295
4296         cur = fs_devices->alloc_list.next;
4297
4298         /*
4299          * in the first pass through the devices list, we gather information
4300          * about the available holes on each device.
4301          */
4302         ndevs = 0;
4303         while (cur != &fs_devices->alloc_list) {
4304                 struct btrfs_device *device;
4305                 u64 max_avail;
4306                 u64 dev_offset;
4307
4308                 device = list_entry(cur, struct btrfs_device, dev_alloc_list);
4309
4310                 cur = cur->next;
4311
4312                 if (!device->writeable) {
4313                         WARN(1, KERN_ERR
4314                                "BTRFS: read-only device in alloc_list\n");
4315                         continue;
4316                 }
4317
4318                 if (!device->in_fs_metadata ||
4319                     device->is_tgtdev_for_dev_replace)
4320                         continue;
4321
4322                 if (device->total_bytes > device->bytes_used)
4323                         total_avail = device->total_bytes - device->bytes_used;
4324                 else
4325                         total_avail = 0;
4326
4327                 /* If there is no space on this device, skip it. */
4328                 if (total_avail == 0)
4329                         continue;
4330
4331                 ret = find_free_dev_extent(trans, device,
4332                                            max_stripe_size * dev_stripes,
4333                                            &dev_offset, &max_avail);
4334                 if (ret && ret != -ENOSPC)
4335                         goto error;
4336
4337                 if (ret == 0)
4338                         max_avail = max_stripe_size * dev_stripes;
4339
4340                 if (max_avail < BTRFS_STRIPE_LEN * dev_stripes)
4341                         continue;
4342
4343                 if (ndevs == fs_devices->rw_devices) {
4344                         WARN(1, "%s: found more than %llu devices\n",
4345                              __func__, fs_devices->rw_devices);
4346                         break;
4347                 }
4348                 devices_info[ndevs].dev_offset = dev_offset;
4349                 devices_info[ndevs].max_avail = max_avail;
4350                 devices_info[ndevs].total_avail = total_avail;
4351                 devices_info[ndevs].dev = device;
4352                 ++ndevs;
4353         }
4354
4355         /*
4356          * now sort the devices by hole size / available space
4357          */
4358         sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
4359              btrfs_cmp_device_info, NULL);
4360
4361         /* round down to number of usable stripes */
4362         ndevs -= ndevs % devs_increment;
4363
4364         if (ndevs < devs_increment * sub_stripes || ndevs < devs_min) {
4365                 ret = -ENOSPC;
4366                 goto error;
4367         }
4368
4369         if (devs_max && ndevs > devs_max)
4370                 ndevs = devs_max;
4371         /*
4372          * the primary goal is to maximize the number of stripes, so use as many
4373          * devices as possible, even if the stripes are not maximum sized.
4374          */
4375         stripe_size = devices_info[ndevs-1].max_avail;
4376         num_stripes = ndevs * dev_stripes;
4377
4378         /*
4379          * this will have to be fixed for RAID1 and RAID10 over
4380          * more drives
4381          */
4382         data_stripes = num_stripes / ncopies;
4383
4384         if (type & BTRFS_BLOCK_GROUP_RAID5) {
4385                 raid_stripe_len = find_raid56_stripe_len(ndevs - 1,
4386                                  btrfs_super_stripesize(info->super_copy));
4387                 data_stripes = num_stripes - 1;
4388         }
4389         if (type & BTRFS_BLOCK_GROUP_RAID6) {
4390                 raid_stripe_len = find_raid56_stripe_len(ndevs - 2,
4391                                  btrfs_super_stripesize(info->super_copy));
4392                 data_stripes = num_stripes - 2;
4393         }
4394
4395         /*
4396          * Use the number of data stripes to figure out how big this chunk
4397          * is really going to be in terms of logical address space,
4398          * and compare that answer with the max chunk size
4399          */
4400         if (stripe_size * data_stripes > max_chunk_size) {
4401                 u64 mask = (1ULL << 24) - 1;
4402
4403                 stripe_size = div_u64(max_chunk_size, data_stripes);
4404
4405                 /* bump the answer up to a 16MB boundary */
4406                 stripe_size = (stripe_size + mask) & ~mask;
4407
4408                 /* but don't go higher than the limits we found
4409                  * while searching for free extents
4410                  */
4411                 if (stripe_size > devices_info[ndevs-1].max_avail)
4412                         stripe_size = devices_info[ndevs-1].max_avail;
4413         }
4414
4415         stripe_size = div_u64(stripe_size, dev_stripes);
4416
4417         /* align to BTRFS_STRIPE_LEN */
4418         stripe_size = div_u64(stripe_size, raid_stripe_len);
4419         stripe_size *= raid_stripe_len;
4420
4421         map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
4422         if (!map) {
4423                 ret = -ENOMEM;
4424                 goto error;
4425         }
4426         map->num_stripes = num_stripes;
4427
4428         for (i = 0; i < ndevs; ++i) {
4429                 for (j = 0; j < dev_stripes; ++j) {
4430                         int s = i * dev_stripes + j;
4431                         map->stripes[s].dev = devices_info[i].dev;
4432                         map->stripes[s].physical = devices_info[i].dev_offset +
4433                                                    j * stripe_size;
4434                 }
4435         }
4436         map->sector_size = extent_root->sectorsize;
4437         map->stripe_len = raid_stripe_len;
4438         map->io_align = raid_stripe_len;
4439         map->io_width = raid_stripe_len;
4440         map->type = type;
4441         map->sub_stripes = sub_stripes;
4442
4443         num_bytes = stripe_size * data_stripes;
4444
4445         trace_btrfs_chunk_alloc(info->chunk_root, map, start, num_bytes);
4446
4447         em = alloc_extent_map();
4448         if (!em) {
4449                 kfree(map);
4450                 ret = -ENOMEM;
4451                 goto error;
4452         }
4453         set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
4454         em->bdev = (struct block_device *)map;
4455         em->start = start;
4456         em->len = num_bytes;
4457         em->block_start = 0;
4458         em->block_len = em->len;
4459         em->orig_block_len = stripe_size;
4460
4461         em_tree = &extent_root->fs_info->mapping_tree.map_tree;
4462         write_lock(&em_tree->lock);
4463         ret = add_extent_mapping(em_tree, em, 0);
4464         if (!ret) {
4465                 list_add_tail(&em->list, &trans->transaction->pending_chunks);
4466                 atomic_inc(&em->refs);
4467         }
4468         write_unlock(&em_tree->lock);
4469         if (ret) {
4470                 free_extent_map(em);
4471                 goto error;
4472         }
4473
4474         ret = btrfs_make_block_group(trans, extent_root, 0, type,
4475                                      BTRFS_FIRST_CHUNK_TREE_OBJECTID,
4476                                      start, num_bytes);
4477         if (ret)
4478                 goto error_del_extent;
4479
4480         for (i = 0; i < map->num_stripes; i++) {
4481                 num_bytes = map->stripes[i].dev->bytes_used + stripe_size;
4482                 btrfs_device_set_bytes_used(map->stripes[i].dev, num_bytes);
4483         }
4484
4485         spin_lock(&extent_root->fs_info->free_chunk_lock);
4486         extent_root->fs_info->free_chunk_space -= (stripe_size *
4487                                                    map->num_stripes);
4488         spin_unlock(&extent_root->fs_info->free_chunk_lock);
4489
4490         free_extent_map(em);
4491         check_raid56_incompat_flag(extent_root->fs_info, type);
4492
4493         kfree(devices_info);
4494         return 0;
4495
4496 error_del_extent:
4497         write_lock(&em_tree->lock);
4498         remove_extent_mapping(em_tree, em);
4499         write_unlock(&em_tree->lock);
4500
4501         /* One for our allocation */
4502         free_extent_map(em);
4503         /* One for the tree reference */
4504         free_extent_map(em);
4505         /* One for the pending_chunks list reference */
4506         free_extent_map(em);
4507 error:
4508         kfree(devices_info);
4509         return ret;
4510 }
4511
4512 int btrfs_finish_chunk_alloc(struct btrfs_trans_handle *trans,
4513                                 struct btrfs_root *extent_root,
4514                                 u64 chunk_offset, u64 chunk_size)
4515 {
4516         struct btrfs_key key;
4517         struct btrfs_root *chunk_root = extent_root->fs_info->chunk_root;
4518         struct btrfs_device *device;
4519         struct btrfs_chunk *chunk;
4520         struct btrfs_stripe *stripe;
4521         struct extent_map_tree *em_tree;
4522         struct extent_map *em;
4523         struct map_lookup *map;
4524         size_t item_size;
4525         u64 dev_offset;
4526         u64 stripe_size;
4527         int i = 0;
4528         int ret;
4529
4530         em_tree = &extent_root->fs_info->mapping_tree.map_tree;
4531         read_lock(&em_tree->lock);
4532         em = lookup_extent_mapping(em_tree, chunk_offset, chunk_size);
4533         read_unlock(&em_tree->lock);
4534
4535         if (!em) {
4536                 btrfs_crit(extent_root->fs_info, "unable to find logical "
4537                            "%Lu len %Lu", chunk_offset, chunk_size);
4538                 return -EINVAL;
4539         }
4540
4541         if (em->start != chunk_offset || em->len != chunk_size) {
4542                 btrfs_crit(extent_root->fs_info, "found a bad mapping, wanted"
4543                           " %Lu-%Lu, found %Lu-%Lu", chunk_offset,
4544                           chunk_size, em->start, em->len);
4545                 free_extent_map(em);
4546                 return -EINVAL;
4547         }
4548
4549         map = (struct map_lookup *)em->bdev;
4550         item_size = btrfs_chunk_item_size(map->num_stripes);
4551         stripe_size = em->orig_block_len;
4552
4553         chunk = kzalloc(item_size, GFP_NOFS);
4554         if (!chunk) {
4555                 ret = -ENOMEM;
4556                 goto out;
4557         }
4558
4559         for (i = 0; i < map->num_stripes; i++) {
4560                 device = map->stripes[i].dev;
4561                 dev_offset = map->stripes[i].physical;
4562
4563                 ret = btrfs_update_device(trans, device);
4564                 if (ret)
4565                         goto out;
4566                 ret = btrfs_alloc_dev_extent(trans, device,
4567                                              chunk_root->root_key.objectid,
4568                                              BTRFS_FIRST_CHUNK_TREE_OBJECTID,
4569                                              chunk_offset, dev_offset,
4570                                              stripe_size);
4571                 if (ret)
4572                         goto out;
4573         }
4574
4575         stripe = &chunk->stripe;
4576         for (i = 0; i < map->num_stripes; i++) {
4577                 device = map->stripes[i].dev;
4578                 dev_offset = map->stripes[i].physical;
4579
4580                 btrfs_set_stack_stripe_devid(stripe, device->devid);
4581                 btrfs_set_stack_stripe_offset(stripe, dev_offset);
4582                 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
4583                 stripe++;
4584         }
4585
4586         btrfs_set_stack_chunk_length(chunk, chunk_size);
4587         btrfs_set_stack_chunk_owner(chunk, extent_root->root_key.objectid);
4588         btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
4589         btrfs_set_stack_chunk_type(chunk, map->type);
4590         btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
4591         btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
4592         btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
4593         btrfs_set_stack_chunk_sector_size(chunk, extent_root->sectorsize);
4594         btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
4595
4596         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
4597         key.type = BTRFS_CHUNK_ITEM_KEY;
4598         key.offset = chunk_offset;
4599
4600         ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
4601         if (ret == 0 && map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
4602                 /*
4603                  * TODO: Cleanup of inserted chunk root in case of
4604                  * failure.
4605                  */
4606                 ret = btrfs_add_system_chunk(chunk_root, &key, chunk,
4607                                              item_size);
4608         }
4609
4610 out:
4611         kfree(chunk);
4612         free_extent_map(em);
4613         return ret;
4614 }
4615
4616 /*
4617  * Chunk allocation falls into two parts. The first part does works
4618  * that make the new allocated chunk useable, but not do any operation
4619  * that modifies the chunk tree. The second part does the works that
4620  * require modifying the chunk tree. This division is important for the
4621  * bootstrap process of adding storage to a seed btrfs.
4622  */
4623 int btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
4624                       struct btrfs_root *extent_root, u64 type)
4625 {
4626         u64 chunk_offset;
4627
4628         ASSERT(mutex_is_locked(&extent_root->fs_info->chunk_mutex));
4629         chunk_offset = find_next_chunk(extent_root->fs_info);
4630         return __btrfs_alloc_chunk(trans, extent_root, chunk_offset, type);
4631 }
4632
4633 static noinline int init_first_rw_device(struct btrfs_trans_handle *trans,
4634                                          struct btrfs_root *root,
4635                                          struct btrfs_device *device)
4636 {
4637         u64 chunk_offset;
4638         u64 sys_chunk_offset;
4639         u64 alloc_profile;
4640         struct btrfs_fs_info *fs_info = root->fs_info;
4641         struct btrfs_root *extent_root = fs_info->extent_root;
4642         int ret;
4643
4644         chunk_offset = find_next_chunk(fs_info);
4645         alloc_profile = btrfs_get_alloc_profile(extent_root, 0);
4646         ret = __btrfs_alloc_chunk(trans, extent_root, chunk_offset,
4647                                   alloc_profile);
4648         if (ret)
4649                 return ret;
4650
4651         sys_chunk_offset = find_next_chunk(root->fs_info);
4652         alloc_profile = btrfs_get_alloc_profile(fs_info->chunk_root, 0);
4653         ret = __btrfs_alloc_chunk(trans, extent_root, sys_chunk_offset,
4654                                   alloc_profile);
4655         return ret;
4656 }
4657
4658 static inline int btrfs_chunk_max_errors(struct map_lookup *map)
4659 {
4660         int max_errors;
4661
4662         if (map->type & (BTRFS_BLOCK_GROUP_RAID1 |
4663                          BTRFS_BLOCK_GROUP_RAID10 |
4664                          BTRFS_BLOCK_GROUP_RAID5 |
4665                          BTRFS_BLOCK_GROUP_DUP)) {
4666                 max_errors = 1;
4667         } else if (map->type & BTRFS_BLOCK_GROUP_RAID6) {
4668                 max_errors = 2;
4669         } else {
4670                 max_errors = 0;
4671         }
4672
4673         return max_errors;
4674 }
4675
4676 int btrfs_chunk_readonly(struct btrfs_root *root, u64 chunk_offset)
4677 {
4678         struct extent_map *em;
4679         struct map_lookup *map;
4680         struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
4681         int readonly = 0;
4682         int miss_ndevs = 0;
4683         int i;
4684
4685         read_lock(&map_tree->map_tree.lock);
4686         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
4687         read_unlock(&map_tree->map_tree.lock);
4688         if (!em)
4689                 return 1;
4690
4691         map = (struct map_lookup *)em->bdev;
4692         for (i = 0; i < map->num_stripes; i++) {
4693                 if (map->stripes[i].dev->missing) {
4694                         miss_ndevs++;
4695                         continue;
4696                 }
4697
4698                 if (!map->stripes[i].dev->writeable) {
4699                         readonly = 1;
4700                         goto end;
4701                 }
4702         }
4703
4704         /*
4705          * If the number of missing devices is larger than max errors,
4706          * we can not write the data into that chunk successfully, so
4707          * set it readonly.
4708          */
4709         if (miss_ndevs > btrfs_chunk_max_errors(map))
4710                 readonly = 1;
4711 end:
4712         free_extent_map(em);
4713         return readonly;
4714 }
4715
4716 void btrfs_mapping_init(struct btrfs_mapping_tree *tree)
4717 {
4718         extent_map_tree_init(&tree->map_tree);
4719 }
4720
4721 void btrfs_mapping_tree_free(struct btrfs_mapping_tree *tree)
4722 {
4723         struct extent_map *em;
4724
4725         while (1) {
4726                 write_lock(&tree->map_tree.lock);
4727                 em = lookup_extent_mapping(&tree->map_tree, 0, (u64)-1);
4728                 if (em)
4729                         remove_extent_mapping(&tree->map_tree, em);
4730                 write_unlock(&tree->map_tree.lock);
4731                 if (!em)
4732                         break;
4733                 /* once for us */
4734                 free_extent_map(em);
4735                 /* once for the tree */
4736                 free_extent_map(em);
4737         }
4738 }
4739
4740 int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
4741 {
4742         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
4743         struct extent_map *em;
4744         struct map_lookup *map;
4745         struct extent_map_tree *em_tree = &map_tree->map_tree;
4746         int ret;
4747
4748         read_lock(&em_tree->lock);
4749         em = lookup_extent_mapping(em_tree, logical, len);
4750         read_unlock(&em_tree->lock);
4751
4752         /*
4753          * We could return errors for these cases, but that could get ugly and
4754          * we'd probably do the same thing which is just not do anything else
4755          * and exit, so return 1 so the callers don't try to use other copies.
4756          */
4757         if (!em) {
4758                 btrfs_crit(fs_info, "No mapping for %Lu-%Lu", logical,
4759                             logical+len);
4760                 return 1;
4761         }
4762
4763         if (em->start > logical || em->start + em->len < logical) {
4764                 btrfs_crit(fs_info, "Invalid mapping for %Lu-%Lu, got "
4765                             "%Lu-%Lu", logical, logical+len, em->start,
4766                             em->start + em->len);
4767                 free_extent_map(em);
4768                 return 1;
4769         }
4770
4771         map = (struct map_lookup *)em->bdev;
4772         if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1))
4773                 ret = map->num_stripes;
4774         else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
4775                 ret = map->sub_stripes;
4776         else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
4777                 ret = 2;
4778         else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
4779                 ret = 3;
4780         else
4781                 ret = 1;
4782         free_extent_map(em);
4783
4784         btrfs_dev_replace_lock(&fs_info->dev_replace);
4785         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))
4786                 ret++;
4787         btrfs_dev_replace_unlock(&fs_info->dev_replace);
4788
4789         return ret;
4790 }
4791
4792 unsigned long btrfs_full_stripe_len(struct btrfs_root *root,
4793                                     struct btrfs_mapping_tree *map_tree,
4794                                     u64 logical)
4795 {
4796         struct extent_map *em;
4797         struct map_lookup *map;
4798         struct extent_map_tree *em_tree = &map_tree->map_tree;
4799         unsigned long len = root->sectorsize;
4800
4801         read_lock(&em_tree->lock);
4802         em = lookup_extent_mapping(em_tree, logical, len);
4803         read_unlock(&em_tree->lock);
4804         BUG_ON(!em);
4805
4806         BUG_ON(em->start > logical || em->start + em->len < logical);
4807         map = (struct map_lookup *)em->bdev;
4808         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
4809                 len = map->stripe_len * nr_data_stripes(map);
4810         free_extent_map(em);
4811         return len;
4812 }
4813
4814 int btrfs_is_parity_mirror(struct btrfs_mapping_tree *map_tree,
4815                            u64 logical, u64 len, int mirror_num)
4816 {
4817         struct extent_map *em;
4818         struct map_lookup *map;
4819         struct extent_map_tree *em_tree = &map_tree->map_tree;
4820         int ret = 0;
4821
4822         read_lock(&em_tree->lock);
4823         em = lookup_extent_mapping(em_tree, logical, len);
4824         read_unlock(&em_tree->lock);
4825         BUG_ON(!em);
4826
4827         BUG_ON(em->start > logical || em->start + em->len < logical);
4828         map = (struct map_lookup *)em->bdev;
4829         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
4830                 ret = 1;
4831         free_extent_map(em);
4832         return ret;
4833 }
4834
4835 static int find_live_mirror(struct btrfs_fs_info *fs_info,
4836                             struct map_lookup *map, int first, int num,
4837                             int optimal, int dev_replace_is_ongoing)
4838 {
4839         int i;
4840         int tolerance;
4841         struct btrfs_device *srcdev;
4842
4843         if (dev_replace_is_ongoing &&
4844             fs_info->dev_replace.cont_reading_from_srcdev_mode ==
4845              BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
4846                 srcdev = fs_info->dev_replace.srcdev;
4847         else
4848                 srcdev = NULL;
4849
4850         /*
4851          * try to avoid the drive that is the source drive for a
4852          * dev-replace procedure, only choose it if no other non-missing
4853          * mirror is available
4854          */
4855         for (tolerance = 0; tolerance < 2; tolerance++) {
4856                 if (map->stripes[optimal].dev->bdev &&
4857                     (tolerance || map->stripes[optimal].dev != srcdev))
4858                         return optimal;
4859                 for (i = first; i < first + num; i++) {
4860                         if (map->stripes[i].dev->bdev &&
4861                             (tolerance || map->stripes[i].dev != srcdev))
4862                                 return i;
4863                 }
4864         }
4865
4866         /* we couldn't find one that doesn't fail.  Just return something
4867          * and the io error handling code will clean up eventually
4868          */
4869         return optimal;
4870 }
4871
4872 static inline int parity_smaller(u64 a, u64 b)
4873 {
4874         return a > b;
4875 }
4876
4877 /* Bubble-sort the stripe set to put the parity/syndrome stripes last */
4878 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)
4879 {
4880         struct btrfs_bio_stripe s;
4881         int i;
4882         u64 l;
4883         int again = 1;
4884
4885         while (again) {
4886                 again = 0;
4887                 for (i = 0; i < num_stripes - 1; i++) {
4888                         if (parity_smaller(bbio->raid_map[i],
4889                                            bbio->raid_map[i+1])) {
4890                                 s = bbio->stripes[i];
4891                                 l = bbio->raid_map[i];
4892                                 bbio->stripes[i] = bbio->stripes[i+1];
4893                                 bbio->raid_map[i] = bbio->raid_map[i+1];
4894                                 bbio->stripes[i+1] = s;
4895                                 bbio->raid_map[i+1] = l;
4896
4897                                 again = 1;
4898                         }
4899                 }
4900         }
4901 }
4902
4903 static struct btrfs_bio *alloc_btrfs_bio(int total_stripes, int real_stripes)
4904 {
4905         struct btrfs_bio *bbio = kzalloc(
4906                  /* the size of the btrfs_bio */
4907                 sizeof(struct btrfs_bio) +
4908                 /* plus the variable array for the stripes */
4909                 sizeof(struct btrfs_bio_stripe) * (total_stripes) +
4910                 /* plus the variable array for the tgt dev */
4911                 sizeof(int) * (real_stripes) +
4912                 /*
4913                  * plus the raid_map, which includes both the tgt dev
4914                  * and the stripes
4915                  */
4916                 sizeof(u64) * (total_stripes),
4917                 GFP_NOFS);
4918         if (!bbio)
4919                 return NULL;
4920
4921         atomic_set(&bbio->error, 0);
4922         atomic_set(&bbio->refs, 1);
4923
4924         return bbio;
4925 }
4926
4927 void btrfs_get_bbio(struct btrfs_bio *bbio)
4928 {
4929         WARN_ON(!atomic_read(&bbio->refs));
4930         atomic_inc(&bbio->refs);
4931 }
4932
4933 void btrfs_put_bbio(struct btrfs_bio *bbio)
4934 {
4935         if (!bbio)
4936                 return;
4937         if (atomic_dec_and_test(&bbio->refs))
4938                 kfree(bbio);
4939 }
4940
4941 static int __btrfs_map_block(struct btrfs_fs_info *fs_info, int rw,
4942                              u64 logical, u64 *length,
4943                              struct btrfs_bio **bbio_ret,
4944                              int mirror_num, int need_raid_map)
4945 {
4946         struct extent_map *em;
4947         struct map_lookup *map;
4948         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
4949         struct extent_map_tree *em_tree = &map_tree->map_tree;
4950         u64 offset;
4951         u64 stripe_offset;
4952         u64 stripe_end_offset;
4953         u64 stripe_nr;
4954         u64 stripe_nr_orig;
4955         u64 stripe_nr_end;
4956         u64 stripe_len;
4957         u32 stripe_index;
4958         int i;
4959         int ret = 0;
4960         int num_stripes;
4961         int max_errors = 0;
4962         int tgtdev_indexes = 0;
4963         struct btrfs_bio *bbio = NULL;
4964         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
4965         int dev_replace_is_ongoing = 0;
4966         int num_alloc_stripes;
4967         int patch_the_first_stripe_for_dev_replace = 0;
4968         u64 physical_to_patch_in_first_stripe = 0;
4969         u64 raid56_full_stripe_start = (u64)-1;
4970
4971         read_lock(&em_tree->lock);
4972         em = lookup_extent_mapping(em_tree, logical, *length);
4973         read_unlock(&em_tree->lock);
4974
4975         if (!em) {
4976                 btrfs_crit(fs_info, "unable to find logical %llu len %llu",
4977                         logical, *length);
4978                 return -EINVAL;
4979         }
4980
4981         if (em->start > logical || em->start + em->len < logical) {
4982                 btrfs_crit(fs_info, "found a bad mapping, wanted %Lu, "
4983                            "found %Lu-%Lu", logical, em->start,
4984                            em->start + em->len);
4985                 free_extent_map(em);
4986                 return -EINVAL;
4987         }
4988
4989         map = (struct map_lookup *)em->bdev;
4990         offset = logical - em->start;
4991
4992         stripe_len = map->stripe_len;
4993         stripe_nr = offset;
4994         /*
4995          * stripe_nr counts the total number of stripes we have to stride
4996          * to get to this block
4997          */
4998         stripe_nr = div64_u64(stripe_nr, stripe_len);
4999
5000         stripe_offset = stripe_nr * stripe_len;
5001         BUG_ON(offset < stripe_offset);
5002
5003         /* stripe_offset is the offset of this block in its stripe*/
5004         stripe_offset = offset - stripe_offset;
5005
5006         /* if we're here for raid56, we need to know the stripe aligned start */
5007         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5008                 unsigned long full_stripe_len = stripe_len * nr_data_stripes(map);
5009                 raid56_full_stripe_start = offset;
5010
5011                 /* allow a write of a full stripe, but make sure we don't
5012                  * allow straddling of stripes
5013                  */
5014                 raid56_full_stripe_start = div64_u64(raid56_full_stripe_start,
5015                                 full_stripe_len);
5016                 raid56_full_stripe_start *= full_stripe_len;
5017         }
5018
5019         if (rw & REQ_DISCARD) {
5020                 /* we don't discard raid56 yet */
5021                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5022                         ret = -EOPNOTSUPP;
5023                         goto out;
5024                 }
5025                 *length = min_t(u64, em->len - offset, *length);
5026         } else if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
5027                 u64 max_len;
5028                 /* For writes to RAID[56], allow a full stripeset across all disks.
5029                    For other RAID types and for RAID[56] reads, just allow a single
5030                    stripe (on a single disk). */
5031                 if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
5032                     (rw & REQ_WRITE)) {
5033                         max_len = stripe_len * nr_data_stripes(map) -
5034                                 (offset - raid56_full_stripe_start);
5035                 } else {
5036                         /* we limit the length of each bio to what fits in a stripe */
5037                         max_len = stripe_len - stripe_offset;
5038                 }
5039                 *length = min_t(u64, em->len - offset, max_len);
5040         } else {
5041                 *length = em->len - offset;
5042         }
5043
5044         /* This is for when we're called from btrfs_merge_bio_hook() and all
5045            it cares about is the length */
5046         if (!bbio_ret)
5047                 goto out;
5048
5049         btrfs_dev_replace_lock(dev_replace);
5050         dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
5051         if (!dev_replace_is_ongoing)
5052                 btrfs_dev_replace_unlock(dev_replace);
5053
5054         if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
5055             !(rw & (REQ_WRITE | REQ_DISCARD | REQ_GET_READ_MIRRORS)) &&
5056             dev_replace->tgtdev != NULL) {
5057                 /*
5058                  * in dev-replace case, for repair case (that's the only
5059                  * case where the mirror is selected explicitly when
5060                  * calling btrfs_map_block), blocks left of the left cursor
5061                  * can also be read from the target drive.
5062                  * For REQ_GET_READ_MIRRORS, the target drive is added as
5063                  * the last one to the array of stripes. For READ, it also
5064                  * needs to be supported using the same mirror number.
5065                  * If the requested block is not left of the left cursor,
5066                  * EIO is returned. This can happen because btrfs_num_copies()
5067                  * returns one more in the dev-replace case.
5068                  */
5069                 u64 tmp_length = *length;
5070                 struct btrfs_bio *tmp_bbio = NULL;
5071                 int tmp_num_stripes;
5072                 u64 srcdev_devid = dev_replace->srcdev->devid;
5073                 int index_srcdev = 0;
5074                 int found = 0;
5075                 u64 physical_of_found = 0;
5076
5077                 ret = __btrfs_map_block(fs_info, REQ_GET_READ_MIRRORS,
5078                              logical, &tmp_length, &tmp_bbio, 0, 0);
5079                 if (ret) {
5080                         WARN_ON(tmp_bbio != NULL);
5081                         goto out;
5082                 }
5083
5084                 tmp_num_stripes = tmp_bbio->num_stripes;
5085                 if (mirror_num > tmp_num_stripes) {
5086                         /*
5087                          * REQ_GET_READ_MIRRORS does not contain this
5088                          * mirror, that means that the requested area
5089                          * is not left of the left cursor
5090                          */
5091                         ret = -EIO;
5092                         btrfs_put_bbio(tmp_bbio);
5093                         goto out;
5094                 }
5095
5096                 /*
5097                  * process the rest of the function using the mirror_num
5098                  * of the source drive. Therefore look it up first.
5099                  * At the end, patch the device pointer to the one of the
5100                  * target drive.
5101                  */
5102                 for (i = 0; i < tmp_num_stripes; i++) {
5103                         if (tmp_bbio->stripes[i].dev->devid == srcdev_devid) {
5104                                 /*
5105                                  * In case of DUP, in order to keep it
5106                                  * simple, only add the mirror with the
5107                                  * lowest physical address
5108                                  */
5109                                 if (found &&
5110                                     physical_of_found <=
5111                                      tmp_bbio->stripes[i].physical)
5112                                         continue;
5113                                 index_srcdev = i;
5114                                 found = 1;
5115                                 physical_of_found =
5116                                         tmp_bbio->stripes[i].physical;
5117                         }
5118                 }
5119
5120                 if (found) {
5121                         mirror_num = index_srcdev + 1;
5122                         patch_the_first_stripe_for_dev_replace = 1;
5123                         physical_to_patch_in_first_stripe = physical_of_found;
5124                 } else {
5125                         WARN_ON(1);
5126                         ret = -EIO;
5127                         btrfs_put_bbio(tmp_bbio);
5128                         goto out;
5129                 }
5130
5131                 btrfs_put_bbio(tmp_bbio);
5132         } else if (mirror_num > map->num_stripes) {
5133                 mirror_num = 0;
5134         }
5135
5136         num_stripes = 1;
5137         stripe_index = 0;
5138         stripe_nr_orig = stripe_nr;
5139         stripe_nr_end = ALIGN(offset + *length, map->stripe_len);
5140         stripe_nr_end = div_u64(stripe_nr_end, map->stripe_len);
5141         stripe_end_offset = stripe_nr_end * map->stripe_len -
5142                             (offset + *length);
5143
5144         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
5145                 if (rw & REQ_DISCARD)
5146                         num_stripes = min_t(u64, map->num_stripes,
5147                                             stripe_nr_end - stripe_nr_orig);
5148                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
5149                                 &stripe_index);
5150                 if (!(rw & (REQ_WRITE | REQ_DISCARD | REQ_GET_READ_MIRRORS)))
5151                         mirror_num = 1;
5152         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
5153                 if (rw & (REQ_WRITE | REQ_DISCARD | REQ_GET_READ_MIRRORS))
5154                         num_stripes = map->num_stripes;
5155                 else if (mirror_num)
5156                         stripe_index = mirror_num - 1;
5157                 else {
5158                         stripe_index = find_live_mirror(fs_info, map, 0,
5159                                             map->num_stripes,
5160                                             current->pid % map->num_stripes,
5161                                             dev_replace_is_ongoing);
5162                         mirror_num = stripe_index + 1;
5163                 }
5164
5165         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
5166                 if (rw & (REQ_WRITE | REQ_DISCARD | REQ_GET_READ_MIRRORS)) {
5167                         num_stripes = map->num_stripes;
5168                 } else if (mirror_num) {
5169                         stripe_index = mirror_num - 1;
5170                 } else {
5171                         mirror_num = 1;
5172                 }
5173
5174         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
5175                 u32 factor = map->num_stripes / map->sub_stripes;
5176
5177                 stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
5178                 stripe_index *= map->sub_stripes;
5179
5180                 if (rw & (REQ_WRITE | REQ_GET_READ_MIRRORS))
5181                         num_stripes = map->sub_stripes;
5182                 else if (rw & REQ_DISCARD)
5183                         num_stripes = min_t(u64, map->sub_stripes *
5184                                             (stripe_nr_end - stripe_nr_orig),
5185                                             map->num_stripes);
5186                 else if (mirror_num)
5187                         stripe_index += mirror_num - 1;
5188                 else {
5189                         int old_stripe_index = stripe_index;
5190                         stripe_index = find_live_mirror(fs_info, map,
5191                                               stripe_index,
5192                                               map->sub_stripes, stripe_index +
5193                                               current->pid % map->sub_stripes,
5194                                               dev_replace_is_ongoing);
5195                         mirror_num = stripe_index - old_stripe_index + 1;
5196                 }
5197
5198         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5199                 if (need_raid_map &&
5200                     ((rw & (REQ_WRITE | REQ_GET_READ_MIRRORS)) ||
5201                      mirror_num > 1)) {
5202                         /* push stripe_nr back to the start of the full stripe */
5203                         stripe_nr = div_u64(raid56_full_stripe_start,
5204                                         stripe_len * nr_data_stripes(map));
5205
5206                         /* RAID[56] write or recovery. Return all stripes */
5207                         num_stripes = map->num_stripes;
5208                         max_errors = nr_parity_stripes(map);
5209
5210                         *length = map->stripe_len;
5211                         stripe_index = 0;
5212                         stripe_offset = 0;
5213                 } else {
5214                         /*
5215                          * Mirror #0 or #1 means the original data block.
5216                          * Mirror #2 is RAID5 parity block.
5217                          * Mirror #3 is RAID6 Q block.
5218                          */
5219                         stripe_nr = div_u64_rem(stripe_nr,
5220                                         nr_data_stripes(map), &stripe_index);
5221                         if (mirror_num > 1)
5222                                 stripe_index = nr_data_stripes(map) +
5223                                                 mirror_num - 2;
5224
5225                         /* We distribute the parity blocks across stripes */
5226                         div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
5227                                         &stripe_index);
5228                         if (!(rw & (REQ_WRITE | REQ_DISCARD |
5229                                     REQ_GET_READ_MIRRORS)) && mirror_num <= 1)
5230                                 mirror_num = 1;
5231                 }
5232         } else {
5233                 /*
5234                  * after this, stripe_nr is the number of stripes on this
5235                  * device we have to walk to find the data, and stripe_index is
5236                  * the number of our device in the stripe array
5237                  */
5238                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
5239                                 &stripe_index);
5240                 mirror_num = stripe_index + 1;
5241         }
5242         BUG_ON(stripe_index >= map->num_stripes);
5243
5244         num_alloc_stripes = num_stripes;
5245         if (dev_replace_is_ongoing) {
5246                 if (rw & (REQ_WRITE | REQ_DISCARD))
5247                         num_alloc_stripes <<= 1;
5248                 if (rw & REQ_GET_READ_MIRRORS)
5249                         num_alloc_stripes++;
5250                 tgtdev_indexes = num_stripes;
5251         }
5252
5253         bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);
5254         if (!bbio) {
5255                 ret = -ENOMEM;
5256                 goto out;
5257         }
5258         if (dev_replace_is_ongoing)
5259                 bbio->tgtdev_map = (int *)(bbio->stripes + num_alloc_stripes);
5260
5261         /* build raid_map */
5262         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK &&
5263             need_raid_map && ((rw & (REQ_WRITE | REQ_GET_READ_MIRRORS)) ||
5264             mirror_num > 1)) {
5265                 u64 tmp;
5266                 unsigned rot;
5267
5268                 bbio->raid_map = (u64 *)((void *)bbio->stripes +
5269                                  sizeof(struct btrfs_bio_stripe) *
5270                                  num_alloc_stripes +
5271                                  sizeof(int) * tgtdev_indexes);
5272
5273                 /* Work out the disk rotation on this stripe-set */
5274                 div_u64_rem(stripe_nr, num_stripes, &rot);
5275
5276                 /* Fill in the logical address of each stripe */
5277                 tmp = stripe_nr * nr_data_stripes(map);
5278                 for (i = 0; i < nr_data_stripes(map); i++)
5279                         bbio->raid_map[(i+rot) % num_stripes] =
5280                                 em->start + (tmp + i) * map->stripe_len;
5281
5282                 bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;
5283                 if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5284                         bbio->raid_map[(i+rot+1) % num_stripes] =
5285                                 RAID6_Q_STRIPE;
5286         }
5287
5288         if (rw & REQ_DISCARD) {
5289                 u32 factor = 0;
5290                 u32 sub_stripes = 0;
5291                 u64 stripes_per_dev = 0;
5292                 u32 remaining_stripes = 0;
5293                 u32 last_stripe = 0;
5294
5295                 if (map->type &
5296                     (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) {
5297                         if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5298                                 sub_stripes = 1;
5299                         else
5300                                 sub_stripes = map->sub_stripes;
5301
5302                         factor = map->num_stripes / sub_stripes;
5303                         stripes_per_dev = div_u64_rem(stripe_nr_end -
5304                                                       stripe_nr_orig,
5305                                                       factor,
5306                                                       &remaining_stripes);
5307                         div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
5308                         last_stripe *= sub_stripes;
5309                 }
5310
5311                 for (i = 0; i < num_stripes; i++) {
5312                         bbio->stripes[i].physical =
5313                                 map->stripes[stripe_index].physical +
5314                                 stripe_offset + stripe_nr * map->stripe_len;
5315                         bbio->stripes[i].dev = map->stripes[stripe_index].dev;
5316
5317                         if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5318                                          BTRFS_BLOCK_GROUP_RAID10)) {
5319                                 bbio->stripes[i].length = stripes_per_dev *
5320                                                           map->stripe_len;
5321
5322                                 if (i / sub_stripes < remaining_stripes)
5323                                         bbio->stripes[i].length +=
5324                                                 map->stripe_len;
5325
5326                                 /*
5327                                  * Special for the first stripe and
5328                                  * the last stripe:
5329                                  *
5330                                  * |-------|...|-------|
5331                                  *     |----------|
5332                                  *    off     end_off
5333                                  */
5334                                 if (i < sub_stripes)
5335                                         bbio->stripes[i].length -=
5336                                                 stripe_offset;
5337
5338                                 if (stripe_index >= last_stripe &&
5339                                     stripe_index <= (last_stripe +
5340                                                      sub_stripes - 1))
5341                                         bbio->stripes[i].length -=
5342                                                 stripe_end_offset;
5343
5344                                 if (i == sub_stripes - 1)
5345                                         stripe_offset = 0;
5346                         } else
5347                                 bbio->stripes[i].length = *length;
5348
5349                         stripe_index++;
5350                         if (stripe_index == map->num_stripes) {
5351                                 /* This could only happen for RAID0/10 */
5352                                 stripe_index = 0;
5353                                 stripe_nr++;
5354                         }
5355                 }
5356         } else {
5357                 for (i = 0; i < num_stripes; i++) {
5358                         bbio->stripes[i].physical =
5359                                 map->stripes[stripe_index].physical +
5360                                 stripe_offset +
5361                                 stripe_nr * map->stripe_len;
5362                         bbio->stripes[i].dev =
5363                                 map->stripes[stripe_index].dev;
5364                         stripe_index++;
5365                 }
5366         }
5367
5368         if (rw & (REQ_WRITE | REQ_GET_READ_MIRRORS))
5369                 max_errors = btrfs_chunk_max_errors(map);
5370
5371         if (bbio->raid_map)
5372                 sort_parity_stripes(bbio, num_stripes);
5373
5374         tgtdev_indexes = 0;
5375         if (dev_replace_is_ongoing && (rw & (REQ_WRITE | REQ_DISCARD)) &&
5376             dev_replace->tgtdev != NULL) {
5377                 int index_where_to_add;
5378                 u64 srcdev_devid = dev_replace->srcdev->devid;
5379
5380                 /*
5381                  * duplicate the write operations while the dev replace
5382                  * procedure is running. Since the copying of the old disk
5383                  * to the new disk takes place at run time while the
5384                  * filesystem is mounted writable, the regular write
5385                  * operations to the old disk have to be duplicated to go
5386                  * to the new disk as well.
5387                  * Note that device->missing is handled by the caller, and
5388                  * that the write to the old disk is already set up in the
5389                  * stripes array.
5390                  */
5391                 index_where_to_add = num_stripes;
5392                 for (i = 0; i < num_stripes; i++) {
5393                         if (bbio->stripes[i].dev->devid == srcdev_devid) {
5394                                 /* write to new disk, too */
5395                                 struct btrfs_bio_stripe *new =
5396                                         bbio->stripes + index_where_to_add;
5397                                 struct btrfs_bio_stripe *old =
5398                                         bbio->stripes + i;
5399
5400                                 new->physical = old->physical;
5401                                 new->length = old->length;
5402                                 new->dev = dev_replace->tgtdev;
5403                                 bbio->tgtdev_map[i] = index_where_to_add;
5404                                 index_where_to_add++;
5405                                 max_errors++;
5406                                 tgtdev_indexes++;
5407                         }
5408                 }
5409                 num_stripes = index_where_to_add;
5410         } else if (dev_replace_is_ongoing && (rw & REQ_GET_READ_MIRRORS) &&
5411                    dev_replace->tgtdev != NULL) {
5412                 u64 srcdev_devid = dev_replace->srcdev->devid;
5413                 int index_srcdev = 0;
5414                 int found = 0;
5415                 u64 physical_of_found = 0;
5416
5417                 /*
5418                  * During the dev-replace procedure, the target drive can
5419                  * also be used to read data in case it is needed to repair
5420                  * a corrupt block elsewhere. This is possible if the
5421                  * requested area is left of the left cursor. In this area,
5422                  * the target drive is a full copy of the source drive.
5423                  */
5424                 for (i = 0; i < num_stripes; i++) {
5425                         if (bbio->stripes[i].dev->devid == srcdev_devid) {
5426                                 /*
5427                                  * In case of DUP, in order to keep it
5428                                  * simple, only add the mirror with the
5429                                  * lowest physical address
5430                                  */
5431                                 if (found &&
5432                                     physical_of_found <=
5433                                      bbio->stripes[i].physical)
5434                                         continue;
5435                                 index_srcdev = i;
5436                                 found = 1;
5437                                 physical_of_found = bbio->stripes[i].physical;
5438                         }
5439                 }
5440                 if (found) {
5441                         if (physical_of_found + map->stripe_len <=
5442                             dev_replace->cursor_left) {
5443                                 struct btrfs_bio_stripe *tgtdev_stripe =
5444                                         bbio->stripes + num_stripes;
5445
5446                                 tgtdev_stripe->physical = physical_of_found;
5447                                 tgtdev_stripe->length =
5448                                         bbio->stripes[index_srcdev].length;
5449                                 tgtdev_stripe->dev = dev_replace->tgtdev;
5450                                 bbio->tgtdev_map[index_srcdev] = num_stripes;
5451
5452                                 tgtdev_indexes++;
5453                                 num_stripes++;
5454                         }
5455                 }
5456         }
5457
5458         *bbio_ret = bbio;
5459         bbio->map_type = map->type;
5460         bbio->num_stripes = num_stripes;
5461         bbio->max_errors = max_errors;
5462         bbio->mirror_num = mirror_num;
5463         bbio->num_tgtdevs = tgtdev_indexes;
5464
5465         /*
5466          * this is the case that REQ_READ && dev_replace_is_ongoing &&
5467          * mirror_num == num_stripes + 1 && dev_replace target drive is
5468          * available as a mirror
5469          */
5470         if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
5471                 WARN_ON(num_stripes > 1);
5472                 bbio->stripes[0].dev = dev_replace->tgtdev;
5473                 bbio->stripes[0].physical = physical_to_patch_in_first_stripe;
5474                 bbio->mirror_num = map->num_stripes + 1;
5475         }
5476 out:
5477         if (dev_replace_is_ongoing)
5478                 btrfs_dev_replace_unlock(dev_replace);
5479         free_extent_map(em);
5480         return ret;
5481 }
5482
5483 int btrfs_map_block(struct btrfs_fs_info *fs_info, int rw,
5484                       u64 logical, u64 *length,
5485                       struct btrfs_bio **bbio_ret, int mirror_num)
5486 {
5487         return __btrfs_map_block(fs_info, rw, logical, length, bbio_ret,
5488                                  mirror_num, 0);
5489 }
5490
5491 /* For Scrub/replace */
5492 int btrfs_map_sblock(struct btrfs_fs_info *fs_info, int rw,
5493                      u64 logical, u64 *length,
5494                      struct btrfs_bio **bbio_ret, int mirror_num,
5495                      int need_raid_map)
5496 {
5497         return __btrfs_map_block(fs_info, rw, logical, length, bbio_ret,
5498                                  mirror_num, need_raid_map);
5499 }
5500
5501 int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree,
5502                      u64 chunk_start, u64 physical, u64 devid,
5503                      u64 **logical, int *naddrs, int *stripe_len)
5504 {
5505         struct extent_map_tree *em_tree = &map_tree->map_tree;
5506         struct extent_map *em;
5507         struct map_lookup *map;
5508         u64 *buf;
5509         u64 bytenr;
5510         u64 length;
5511         u64 stripe_nr;
5512         u64 rmap_len;
5513         int i, j, nr = 0;
5514
5515         read_lock(&em_tree->lock);
5516         em = lookup_extent_mapping(em_tree, chunk_start, 1);
5517         read_unlock(&em_tree->lock);
5518
5519         if (!em) {
5520                 printk(KERN_ERR "BTRFS: couldn't find em for chunk %Lu\n",
5521                        chunk_start);
5522                 return -EIO;
5523         }
5524
5525         if (em->start != chunk_start) {
5526                 printk(KERN_ERR "BTRFS: bad chunk start, em=%Lu, wanted=%Lu\n",
5527                        em->start, chunk_start);
5528                 free_extent_map(em);
5529                 return -EIO;
5530         }
5531         map = (struct map_lookup *)em->bdev;
5532
5533         length = em->len;
5534         rmap_len = map->stripe_len;
5535
5536         if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5537                 length = div_u64(length, map->num_stripes / map->sub_stripes);
5538         else if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5539                 length = div_u64(length, map->num_stripes);
5540         else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5541                 length = div_u64(length, nr_data_stripes(map));
5542                 rmap_len = map->stripe_len * nr_data_stripes(map);
5543         }
5544
5545         buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
5546         BUG_ON(!buf); /* -ENOMEM */
5547
5548         for (i = 0; i < map->num_stripes; i++) {
5549                 if (devid && map->stripes[i].dev->devid != devid)
5550                         continue;
5551                 if (map->stripes[i].physical > physical ||
5552                     map->stripes[i].physical + length <= physical)
5553                         continue;
5554
5555                 stripe_nr = physical - map->stripes[i].physical;
5556                 stripe_nr = div_u64(stripe_nr, map->stripe_len);
5557
5558                 if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
5559                         stripe_nr = stripe_nr * map->num_stripes + i;
5560                         stripe_nr = div_u64(stripe_nr, map->sub_stripes);
5561                 } else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
5562                         stripe_nr = stripe_nr * map->num_stripes + i;
5563                 } /* else if RAID[56], multiply by nr_data_stripes().
5564                    * Alternatively, just use rmap_len below instead of
5565                    * map->stripe_len */
5566
5567                 bytenr = chunk_start + stripe_nr * rmap_len;
5568                 WARN_ON(nr >= map->num_stripes);
5569                 for (j = 0; j < nr; j++) {
5570                         if (buf[j] == bytenr)
5571                                 break;
5572                 }
5573                 if (j == nr) {
5574                         WARN_ON(nr >= map->num_stripes);
5575                         buf[nr++] = bytenr;
5576                 }
5577         }
5578
5579         *logical = buf;
5580         *naddrs = nr;
5581         *stripe_len = rmap_len;
5582
5583         free_extent_map(em);
5584         return 0;
5585 }
5586
5587 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio, int err)
5588 {
5589         if (likely(bbio->flags & BTRFS_BIO_ORIG_BIO_SUBMITTED))
5590                 bio_endio_nodec(bio, err);
5591         else
5592                 bio_endio(bio, err);
5593         btrfs_put_bbio(bbio);
5594 }
5595
5596 static void btrfs_end_bio(struct bio *bio, int err)
5597 {
5598         struct btrfs_bio *bbio = bio->bi_private;
5599         struct btrfs_device *dev = bbio->stripes[0].dev;
5600         int is_orig_bio = 0;
5601
5602         if (err) {
5603                 atomic_inc(&bbio->error);
5604                 if (err == -EIO || err == -EREMOTEIO) {
5605                         unsigned int stripe_index =
5606                                 btrfs_io_bio(bio)->stripe_index;
5607
5608                         BUG_ON(stripe_index >= bbio->num_stripes);
5609                         dev = bbio->stripes[stripe_index].dev;
5610                         if (dev->bdev) {
5611                                 if (bio->bi_rw & WRITE)
5612                                         btrfs_dev_stat_inc(dev,
5613                                                 BTRFS_DEV_STAT_WRITE_ERRS);
5614                                 else
5615                                         btrfs_dev_stat_inc(dev,
5616                                                 BTRFS_DEV_STAT_READ_ERRS);
5617                                 if ((bio->bi_rw & WRITE_FLUSH) == WRITE_FLUSH)
5618                                         btrfs_dev_stat_inc(dev,
5619                                                 BTRFS_DEV_STAT_FLUSH_ERRS);
5620                                 btrfs_dev_stat_print_on_error(dev);
5621                         }
5622                 }
5623         }
5624
5625         if (bio == bbio->orig_bio)
5626                 is_orig_bio = 1;
5627
5628         btrfs_bio_counter_dec(bbio->fs_info);
5629
5630         if (atomic_dec_and_test(&bbio->stripes_pending)) {
5631                 if (!is_orig_bio) {
5632                         bio_put(bio);
5633                         bio = bbio->orig_bio;
5634                 }
5635
5636                 bio->bi_private = bbio->private;
5637                 bio->bi_end_io = bbio->end_io;
5638                 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5639                 /* only send an error to the higher layers if it is
5640                  * beyond the tolerance of the btrfs bio
5641                  */
5642                 if (atomic_read(&bbio->error) > bbio->max_errors) {
5643                         err = -EIO;
5644                 } else {
5645                         /*
5646                          * this bio is actually up to date, we didn't
5647                          * go over the max number of errors
5648                          */
5649                         set_bit(BIO_UPTODATE, &bio->bi_flags);
5650                         err = 0;
5651                 }
5652
5653                 btrfs_end_bbio(bbio, bio, err);
5654         } else if (!is_orig_bio) {
5655                 bio_put(bio);
5656         }
5657 }
5658
5659 /*
5660  * see run_scheduled_bios for a description of why bios are collected for
5661  * async submit.
5662  *
5663  * This will add one bio to the pending list for a device and make sure
5664  * the work struct is scheduled.
5665  */
5666 static noinline void btrfs_schedule_bio(struct btrfs_root *root,
5667                                         struct btrfs_device *device,
5668                                         int rw, struct bio *bio)
5669 {
5670         int should_queue = 1;
5671         struct btrfs_pending_bios *pending_bios;
5672
5673         if (device->missing || !device->bdev) {
5674                 bio_endio(bio, -EIO);
5675                 return;
5676         }
5677
5678         /* don't bother with additional async steps for reads, right now */
5679         if (!(rw & REQ_WRITE)) {
5680                 bio_get(bio);
5681                 btrfsic_submit_bio(rw, bio);
5682                 bio_put(bio);
5683                 return;
5684         }
5685
5686         /*
5687          * nr_async_bios allows us to reliably return congestion to the
5688          * higher layers.  Otherwise, the async bio makes it appear we have
5689          * made progress against dirty pages when we've really just put it
5690          * on a queue for later
5691          */
5692         atomic_inc(&root->fs_info->nr_async_bios);
5693         WARN_ON(bio->bi_next);
5694         bio->bi_next = NULL;
5695         bio->bi_rw |= rw;
5696
5697         spin_lock(&device->io_lock);
5698         if (bio->bi_rw & REQ_SYNC)
5699                 pending_bios = &device->pending_sync_bios;
5700         else
5701                 pending_bios = &device->pending_bios;
5702
5703         if (pending_bios->tail)
5704                 pending_bios->tail->bi_next = bio;
5705
5706         pending_bios->tail = bio;
5707         if (!pending_bios->head)
5708                 pending_bios->head = bio;
5709         if (device->running_pending)
5710                 should_queue = 0;
5711
5712         spin_unlock(&device->io_lock);
5713
5714         if (should_queue)
5715                 btrfs_queue_work(root->fs_info->submit_workers,
5716                                  &device->work);
5717 }
5718
5719 static int bio_size_ok(struct block_device *bdev, struct bio *bio,
5720                        sector_t sector)
5721 {
5722         struct bio_vec *prev;
5723         struct request_queue *q = bdev_get_queue(bdev);
5724         unsigned int max_sectors = queue_max_sectors(q);
5725         struct bvec_merge_data bvm = {
5726                 .bi_bdev = bdev,
5727                 .bi_sector = sector,
5728                 .bi_rw = bio->bi_rw,
5729         };
5730
5731         if (WARN_ON(bio->bi_vcnt == 0))
5732                 return 1;
5733
5734         prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
5735         if (bio_sectors(bio) > max_sectors)
5736                 return 0;
5737
5738         if (!q->merge_bvec_fn)
5739                 return 1;
5740
5741         bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len;
5742         if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len)
5743                 return 0;
5744         return 1;
5745 }
5746
5747 static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5748                               struct bio *bio, u64 physical, int dev_nr,
5749                               int rw, int async)
5750 {
5751         struct btrfs_device *dev = bbio->stripes[dev_nr].dev;
5752
5753         bio->bi_private = bbio;
5754         btrfs_io_bio(bio)->stripe_index = dev_nr;
5755         bio->bi_end_io = btrfs_end_bio;
5756         bio->bi_iter.bi_sector = physical >> 9;
5757 #ifdef DEBUG
5758         {
5759                 struct rcu_string *name;
5760
5761                 rcu_read_lock();
5762                 name = rcu_dereference(dev->name);
5763                 pr_debug("btrfs_map_bio: rw %d, sector=%llu, dev=%lu "
5764                          "(%s id %llu), size=%u\n", rw,
5765                          (u64)bio->bi_iter.bi_sector, (u_long)dev->bdev->bd_dev,
5766                          name->str, dev->devid, bio->bi_iter.bi_size);
5767                 rcu_read_unlock();
5768         }
5769 #endif
5770         bio->bi_bdev = dev->bdev;
5771
5772         btrfs_bio_counter_inc_noblocked(root->fs_info);
5773
5774         if (async)
5775                 btrfs_schedule_bio(root, dev, rw, bio);
5776         else
5777                 btrfsic_submit_bio(rw, bio);
5778 }
5779
5780 static int breakup_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5781                               struct bio *first_bio, struct btrfs_device *dev,
5782                               int dev_nr, int rw, int async)
5783 {
5784         struct bio_vec *bvec = first_bio->bi_io_vec;
5785         struct bio *bio;
5786         int nr_vecs = bio_get_nr_vecs(dev->bdev);
5787         u64 physical = bbio->stripes[dev_nr].physical;
5788
5789 again:
5790         bio = btrfs_bio_alloc(dev->bdev, physical >> 9, nr_vecs, GFP_NOFS);
5791         if (!bio)
5792                 return -ENOMEM;
5793
5794         while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) {
5795                 if (bio_add_page(bio, bvec->bv_page, bvec->bv_len,
5796                                  bvec->bv_offset) < bvec->bv_len) {
5797                         u64 len = bio->bi_iter.bi_size;
5798
5799                         atomic_inc(&bbio->stripes_pending);
5800                         submit_stripe_bio(root, bbio, bio, physical, dev_nr,
5801                                           rw, async);
5802                         physical += len;
5803                         goto again;
5804                 }
5805                 bvec++;
5806         }
5807
5808         submit_stripe_bio(root, bbio, bio, physical, dev_nr, rw, async);
5809         return 0;
5810 }
5811
5812 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
5813 {
5814         atomic_inc(&bbio->error);
5815         if (atomic_dec_and_test(&bbio->stripes_pending)) {
5816                 /* Shoud be the original bio. */
5817                 WARN_ON(bio != bbio->orig_bio);
5818
5819                 bio->bi_private = bbio->private;
5820                 bio->bi_end_io = bbio->end_io;
5821                 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5822                 bio->bi_iter.bi_sector = logical >> 9;
5823
5824                 btrfs_end_bbio(bbio, bio, -EIO);
5825         }
5826 }
5827
5828 int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
5829                   int mirror_num, int async_submit)
5830 {
5831         struct btrfs_device *dev;
5832         struct bio *first_bio = bio;
5833         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
5834         u64 length = 0;
5835         u64 map_length;
5836         int ret;
5837         int dev_nr;
5838         int total_devs;
5839         struct btrfs_bio *bbio = NULL;
5840
5841         length = bio->bi_iter.bi_size;
5842         map_length = length;
5843
5844         btrfs_bio_counter_inc_blocked(root->fs_info);
5845         ret = __btrfs_map_block(root->fs_info, rw, logical, &map_length, &bbio,
5846                               mirror_num, 1);
5847         if (ret) {
5848                 btrfs_bio_counter_dec(root->fs_info);
5849                 return ret;
5850         }
5851
5852         total_devs = bbio->num_stripes;
5853         bbio->orig_bio = first_bio;
5854         bbio->private = first_bio->bi_private;
5855         bbio->end_io = first_bio->bi_end_io;
5856         bbio->fs_info = root->fs_info;
5857         atomic_set(&bbio->stripes_pending, bbio->num_stripes);
5858
5859         if (bbio->raid_map) {
5860                 /* In this case, map_length has been set to the length of
5861                    a single stripe; not the whole write */
5862                 if (rw & WRITE) {
5863                         ret = raid56_parity_write(root, bio, bbio, map_length);
5864                 } else {
5865                         ret = raid56_parity_recover(root, bio, bbio, map_length,
5866                                                     mirror_num, 1);
5867                 }
5868
5869                 btrfs_bio_counter_dec(root->fs_info);
5870                 return ret;
5871         }
5872
5873         if (map_length < length) {
5874                 btrfs_crit(root->fs_info, "mapping failed logical %llu bio len %llu len %llu",
5875                         logical, length, map_length);
5876                 BUG();
5877         }
5878
5879         for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
5880                 dev = bbio->stripes[dev_nr].dev;
5881                 if (!dev || !dev->bdev || (rw & WRITE && !dev->writeable)) {
5882                         bbio_error(bbio, first_bio, logical);
5883                         continue;
5884                 }
5885
5886                 /*
5887                  * Check and see if we're ok with this bio based on it's size
5888                  * and offset with the given device.
5889                  */
5890                 if (!bio_size_ok(dev->bdev, first_bio,
5891                                  bbio->stripes[dev_nr].physical >> 9)) {
5892                         ret = breakup_stripe_bio(root, bbio, first_bio, dev,
5893                                                  dev_nr, rw, async_submit);
5894                         BUG_ON(ret);
5895                         continue;
5896                 }
5897
5898                 if (dev_nr < total_devs - 1) {
5899                         bio = btrfs_bio_clone(first_bio, GFP_NOFS);
5900                         BUG_ON(!bio); /* -ENOMEM */
5901                 } else {
5902                         bio = first_bio;
5903                         bbio->flags |= BTRFS_BIO_ORIG_BIO_SUBMITTED;
5904                 }
5905
5906                 submit_stripe_bio(root, bbio, bio,
5907                                   bbio->stripes[dev_nr].physical, dev_nr, rw,
5908                                   async_submit);
5909         }
5910         btrfs_bio_counter_dec(root->fs_info);
5911         return 0;
5912 }
5913
5914 struct btrfs_device *btrfs_find_device(struct btrfs_fs_info *fs_info, u64 devid,
5915                                        u8 *uuid, u8 *fsid)
5916 {
5917         struct btrfs_device *device;
5918         struct btrfs_fs_devices *cur_devices;
5919
5920         cur_devices = fs_info->fs_devices;
5921         while (cur_devices) {
5922                 if (!fsid ||
5923                     !memcmp(cur_devices->fsid, fsid, BTRFS_UUID_SIZE)) {
5924                         device = __find_device(&cur_devices->devices,
5925                                                devid, uuid);
5926                         if (device)
5927                                 return device;
5928                 }
5929                 cur_devices = cur_devices->seed;
5930         }
5931         return NULL;
5932 }
5933
5934 static struct btrfs_device *add_missing_dev(struct btrfs_root *root,
5935                                             struct btrfs_fs_devices *fs_devices,
5936                                             u64 devid, u8 *dev_uuid)
5937 {
5938         struct btrfs_device *device;
5939
5940         device = btrfs_alloc_device(NULL, &devid, dev_uuid);
5941         if (IS_ERR(device))
5942                 return NULL;
5943
5944         list_add(&device->dev_list, &fs_devices->devices);
5945         device->fs_devices = fs_devices;
5946         fs_devices->num_devices++;
5947
5948         device->missing = 1;
5949         fs_devices->missing_devices++;
5950
5951         return device;
5952 }
5953
5954 /**
5955  * btrfs_alloc_device - allocate struct btrfs_device
5956  * @fs_info:    used only for generating a new devid, can be NULL if
5957  *              devid is provided (i.e. @devid != NULL).
5958  * @devid:      a pointer to devid for this device.  If NULL a new devid
5959  *              is generated.
5960  * @uuid:       a pointer to UUID for this device.  If NULL a new UUID
5961  *              is generated.
5962  *
5963  * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
5964  * on error.  Returned struct is not linked onto any lists and can be
5965  * destroyed with kfree() right away.
5966  */
5967 struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
5968                                         const u64 *devid,
5969                                         const u8 *uuid)
5970 {
5971         struct btrfs_device *dev;
5972         u64 tmp;
5973
5974         if (WARN_ON(!devid && !fs_info))
5975                 return ERR_PTR(-EINVAL);
5976
5977         dev = __alloc_device();
5978         if (IS_ERR(dev))
5979                 return dev;
5980
5981         if (devid)
5982                 tmp = *devid;
5983         else {
5984                 int ret;
5985
5986                 ret = find_next_devid(fs_info, &tmp);
5987                 if (ret) {
5988                         kfree(dev);
5989                         return ERR_PTR(ret);
5990                 }
5991         }
5992         dev->devid = tmp;
5993
5994         if (uuid)
5995                 memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
5996         else
5997                 generate_random_uuid(dev->uuid);
5998
5999         btrfs_init_work(&dev->work, btrfs_submit_helper,
6000                         pending_bios_fn, NULL, NULL);
6001
6002         return dev;
6003 }
6004
6005 static int read_one_chunk(struct btrfs_root *root, struct btrfs_key *key,
6006                           struct extent_buffer *leaf,
6007                           struct btrfs_chunk *chunk)
6008 {
6009         struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
6010         struct map_lookup *map;
6011         struct extent_map *em;
6012         u64 logical;
6013         u64 length;
6014         u64 devid;
6015         u8 uuid[BTRFS_UUID_SIZE];
6016         int num_stripes;
6017         int ret;
6018         int i;
6019
6020         logical = key->offset;
6021         length = btrfs_chunk_length(leaf, chunk);
6022
6023         read_lock(&map_tree->map_tree.lock);
6024         em = lookup_extent_mapping(&map_tree->map_tree, logical, 1);
6025         read_unlock(&map_tree->map_tree.lock);
6026
6027         /* already mapped? */
6028         if (em && em->start <= logical && em->start + em->len > logical) {
6029                 free_extent_map(em);
6030                 return 0;
6031         } else if (em) {
6032                 free_extent_map(em);
6033         }
6034
6035         em = alloc_extent_map();
6036         if (!em)
6037                 return -ENOMEM;
6038         num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
6039         map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
6040         if (!map) {
6041                 free_extent_map(em);
6042                 return -ENOMEM;
6043         }
6044
6045         set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
6046         em->bdev = (struct block_device *)map;
6047         em->start = logical;
6048         em->len = length;
6049         em->orig_start = 0;
6050         em->block_start = 0;
6051         em->block_len = em->len;
6052
6053         map->num_stripes = num_stripes;
6054         map->io_width = btrfs_chunk_io_width(leaf, chunk);
6055         map->io_align = btrfs_chunk_io_align(leaf, chunk);
6056         map->sector_size = btrfs_chunk_sector_size(leaf, chunk);
6057         map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
6058         map->type = btrfs_chunk_type(leaf, chunk);
6059         map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
6060         for (i = 0; i < num_stripes; i++) {
6061                 map->stripes[i].physical =
6062                         btrfs_stripe_offset_nr(leaf, chunk, i);
6063                 devid = btrfs_stripe_devid_nr(leaf, chunk, i);
6064                 read_extent_buffer(leaf, uuid, (unsigned long)
6065                                    btrfs_stripe_dev_uuid_nr(chunk, i),
6066                                    BTRFS_UUID_SIZE);
6067                 map->stripes[i].dev = btrfs_find_device(root->fs_info, devid,
6068                                                         uuid, NULL);
6069                 if (!map->stripes[i].dev && !btrfs_test_opt(root, DEGRADED)) {
6070                         free_extent_map(em);
6071                         return -EIO;
6072                 }
6073                 if (!map->stripes[i].dev) {
6074                         map->stripes[i].dev =
6075                                 add_missing_dev(root, root->fs_info->fs_devices,
6076                                                 devid, uuid);
6077                         if (!map->stripes[i].dev) {
6078                                 free_extent_map(em);
6079                                 return -EIO;
6080                         }
6081                 }
6082                 map->stripes[i].dev->in_fs_metadata = 1;
6083         }
6084
6085         write_lock(&map_tree->map_tree.lock);
6086         ret = add_extent_mapping(&map_tree->map_tree, em, 0);
6087         write_unlock(&map_tree->map_tree.lock);
6088         BUG_ON(ret); /* Tree corruption */
6089         free_extent_map(em);
6090
6091         return 0;
6092 }
6093
6094 static void fill_device_from_item(struct extent_buffer *leaf,
6095                                  struct btrfs_dev_item *dev_item,
6096                                  struct btrfs_device *device)
6097 {
6098         unsigned long ptr;
6099
6100         device->devid = btrfs_device_id(leaf, dev_item);
6101         device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
6102         device->total_bytes = device->disk_total_bytes;
6103         device->commit_total_bytes = device->disk_total_bytes;
6104         device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
6105         device->commit_bytes_used = device->bytes_used;
6106         device->type = btrfs_device_type(leaf, dev_item);
6107         device->io_align = btrfs_device_io_align(leaf, dev_item);
6108         device->io_width = btrfs_device_io_width(leaf, dev_item);
6109         device->sector_size = btrfs_device_sector_size(leaf, dev_item);
6110         WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
6111         device->is_tgtdev_for_dev_replace = 0;
6112
6113         ptr = btrfs_device_uuid(dev_item);
6114         read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
6115 }
6116
6117 static struct btrfs_fs_devices *open_seed_devices(struct btrfs_root *root,
6118                                                   u8 *fsid)
6119 {
6120         struct btrfs_fs_devices *fs_devices;
6121         int ret;
6122
6123         BUG_ON(!mutex_is_locked(&uuid_mutex));
6124
6125         fs_devices = root->fs_info->fs_devices->seed;
6126         while (fs_devices) {
6127                 if (!memcmp(fs_devices->fsid, fsid, BTRFS_UUID_SIZE))
6128                         return fs_devices;
6129
6130                 fs_devices = fs_devices->seed;
6131         }
6132
6133         fs_devices = find_fsid(fsid);
6134         if (!fs_devices) {
6135                 if (!btrfs_test_opt(root, DEGRADED))
6136                         return ERR_PTR(-ENOENT);
6137
6138                 fs_devices = alloc_fs_devices(fsid);
6139                 if (IS_ERR(fs_devices))
6140                         return fs_devices;
6141
6142                 fs_devices->seeding = 1;
6143                 fs_devices->opened = 1;
6144                 return fs_devices;
6145         }
6146
6147         fs_devices = clone_fs_devices(fs_devices);
6148         if (IS_ERR(fs_devices))
6149                 return fs_devices;
6150
6151         ret = __btrfs_open_devices(fs_devices, FMODE_READ,
6152                                    root->fs_info->bdev_holder);
6153         if (ret) {
6154                 free_fs_devices(fs_devices);
6155                 fs_devices = ERR_PTR(ret);
6156                 goto out;
6157         }
6158
6159         if (!fs_devices->seeding) {
6160                 __btrfs_close_devices(fs_devices);
6161                 free_fs_devices(fs_devices);
6162                 fs_devices = ERR_PTR(-EINVAL);
6163                 goto out;
6164         }
6165
6166         fs_devices->seed = root->fs_info->fs_devices->seed;
6167         root->fs_info->fs_devices->seed = fs_devices;
6168 out:
6169         return fs_devices;
6170 }
6171
6172 static int read_one_dev(struct btrfs_root *root,
6173                         struct extent_buffer *leaf,
6174                         struct btrfs_dev_item *dev_item)
6175 {
6176         struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
6177         struct btrfs_device *device;
6178         u64 devid;
6179         int ret;
6180         u8 fs_uuid[BTRFS_UUID_SIZE];
6181         u8 dev_uuid[BTRFS_UUID_SIZE];
6182
6183         devid = btrfs_device_id(leaf, dev_item);
6184         read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
6185                            BTRFS_UUID_SIZE);
6186         read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
6187                            BTRFS_UUID_SIZE);
6188
6189         if (memcmp(fs_uuid, root->fs_info->fsid, BTRFS_UUID_SIZE)) {
6190                 fs_devices = open_seed_devices(root, fs_uuid);
6191                 if (IS_ERR(fs_devices))
6192                         return PTR_ERR(fs_devices);
6193         }
6194
6195         device = btrfs_find_device(root->fs_info, devid, dev_uuid, fs_uuid);
6196         if (!device) {
6197                 if (!btrfs_test_opt(root, DEGRADED))
6198                         return -EIO;
6199
6200                 btrfs_warn(root->fs_info, "devid %llu missing", devid);
6201                 device = add_missing_dev(root, fs_devices, devid, dev_uuid);
6202                 if (!device)
6203                         return -ENOMEM;
6204         } else {
6205                 if (!device->bdev && !btrfs_test_opt(root, DEGRADED))
6206                         return -EIO;
6207
6208                 if(!device->bdev && !device->missing) {
6209                         /*
6210                          * this happens when a device that was properly setup
6211                          * in the device info lists suddenly goes bad.
6212                          * device->bdev is NULL, and so we have to set
6213                          * device->missing to one here
6214                          */
6215                         device->fs_devices->missing_devices++;
6216                         device->missing = 1;
6217                 }
6218
6219                 /* Move the device to its own fs_devices */
6220                 if (device->fs_devices != fs_devices) {
6221                         ASSERT(device->missing);
6222
6223                         list_move(&device->dev_list, &fs_devices->devices);
6224                         device->fs_devices->num_devices--;
6225                         fs_devices->num_devices++;
6226
6227                         device->fs_devices->missing_devices--;
6228                         fs_devices->missing_devices++;
6229
6230                         device->fs_devices = fs_devices;
6231                 }
6232         }
6233
6234         if (device->fs_devices != root->fs_info->fs_devices) {
6235                 BUG_ON(device->writeable);
6236                 if (device->generation !=
6237                     btrfs_device_generation(leaf, dev_item))
6238                         return -EINVAL;
6239         }
6240
6241         fill_device_from_item(leaf, dev_item, device);
6242         device->in_fs_metadata = 1;
6243         if (device->writeable && !device->is_tgtdev_for_dev_replace) {
6244                 device->fs_devices->total_rw_bytes += device->total_bytes;
6245                 spin_lock(&root->fs_info->free_chunk_lock);
6246                 root->fs_info->free_chunk_space += device->total_bytes -
6247                         device->bytes_used;
6248                 spin_unlock(&root->fs_info->free_chunk_lock);
6249         }
6250         ret = 0;
6251         return ret;
6252 }
6253
6254 int btrfs_read_sys_array(struct btrfs_root *root)
6255 {
6256         struct btrfs_super_block *super_copy = root->fs_info->super_copy;
6257         struct extent_buffer *sb;
6258         struct btrfs_disk_key *disk_key;
6259         struct btrfs_chunk *chunk;
6260         u8 *array_ptr;
6261         unsigned long sb_array_offset;
6262         int ret = 0;
6263         u32 num_stripes;
6264         u32 array_size;
6265         u32 len = 0;
6266         u32 cur_offset;
6267         struct btrfs_key key;
6268
6269         ASSERT(BTRFS_SUPER_INFO_SIZE <= root->nodesize);
6270         /*
6271          * This will create extent buffer of nodesize, superblock size is
6272          * fixed to BTRFS_SUPER_INFO_SIZE. If nodesize > sb size, this will
6273          * overallocate but we can keep it as-is, only the first page is used.
6274          */
6275         sb = btrfs_find_create_tree_block(root, BTRFS_SUPER_INFO_OFFSET);
6276         if (!sb)
6277                 return -ENOMEM;
6278         btrfs_set_buffer_uptodate(sb);
6279         btrfs_set_buffer_lockdep_class(root->root_key.objectid, sb, 0);
6280         /*
6281          * The sb extent buffer is artifical and just used to read the system array.
6282          * btrfs_set_buffer_uptodate() call does not properly mark all it's
6283          * pages up-to-date when the page is larger: extent does not cover the
6284          * whole page and consequently check_page_uptodate does not find all
6285          * the page's extents up-to-date (the hole beyond sb),
6286          * write_extent_buffer then triggers a WARN_ON.
6287          *
6288          * Regular short extents go through mark_extent_buffer_dirty/writeback cycle,
6289          * but sb spans only this function. Add an explicit SetPageUptodate call
6290          * to silence the warning eg. on PowerPC 64.
6291          */
6292         if (PAGE_CACHE_SIZE > BTRFS_SUPER_INFO_SIZE)
6293                 SetPageUptodate(sb->pages[0]);
6294
6295         write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
6296         array_size = btrfs_super_sys_array_size(super_copy);
6297
6298         array_ptr = super_copy->sys_chunk_array;
6299         sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
6300         cur_offset = 0;
6301
6302         while (cur_offset < array_size) {
6303                 disk_key = (struct btrfs_disk_key *)array_ptr;
6304                 len = sizeof(*disk_key);
6305                 if (cur_offset + len > array_size)
6306                         goto out_short_read;
6307
6308                 btrfs_disk_key_to_cpu(&key, disk_key);
6309
6310                 array_ptr += len;
6311                 sb_array_offset += len;
6312                 cur_offset += len;
6313
6314                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
6315                         chunk = (struct btrfs_chunk *)sb_array_offset;
6316                         /*
6317                          * At least one btrfs_chunk with one stripe must be
6318                          * present, exact stripe count check comes afterwards
6319                          */
6320                         len = btrfs_chunk_item_size(1);
6321                         if (cur_offset + len > array_size)
6322                                 goto out_short_read;
6323
6324                         num_stripes = btrfs_chunk_num_stripes(sb, chunk);
6325                         len = btrfs_chunk_item_size(num_stripes);
6326                         if (cur_offset + len > array_size)
6327                                 goto out_short_read;
6328
6329                         ret = read_one_chunk(root, &key, sb, chunk);
6330                         if (ret)
6331                                 break;
6332                 } else {
6333                         ret = -EIO;
6334                         break;
6335                 }
6336                 array_ptr += len;
6337                 sb_array_offset += len;
6338                 cur_offset += len;
6339         }
6340         free_extent_buffer(sb);
6341         return ret;
6342
6343 out_short_read:
6344         printk(KERN_ERR "BTRFS: sys_array too short to read %u bytes at offset %u\n",
6345                         len, cur_offset);
6346         free_extent_buffer(sb);
6347         return -EIO;
6348 }
6349
6350 int btrfs_read_chunk_tree(struct btrfs_root *root)
6351 {
6352         struct btrfs_path *path;
6353         struct extent_buffer *leaf;
6354         struct btrfs_key key;
6355         struct btrfs_key found_key;
6356         int ret;
6357         int slot;
6358
6359         root = root->fs_info->chunk_root;
6360
6361         path = btrfs_alloc_path();
6362         if (!path)
6363                 return -ENOMEM;
6364
6365         mutex_lock(&uuid_mutex);
6366         lock_chunks(root);
6367
6368         /*
6369          * Read all device items, and then all the chunk items. All
6370          * device items are found before any chunk item (their object id
6371          * is smaller than the lowest possible object id for a chunk
6372          * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
6373          */
6374         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
6375         key.offset = 0;
6376         key.type = 0;
6377         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6378         if (ret < 0)
6379                 goto error;
6380         while (1) {
6381                 leaf = path->nodes[0];
6382                 slot = path->slots[0];
6383                 if (slot >= btrfs_header_nritems(leaf)) {
6384                         ret = btrfs_next_leaf(root, path);
6385                         if (ret == 0)
6386                                 continue;
6387                         if (ret < 0)
6388                                 goto error;
6389                         break;
6390                 }
6391                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
6392                 if (found_key.type == BTRFS_DEV_ITEM_KEY) {
6393                         struct btrfs_dev_item *dev_item;
6394                         dev_item = btrfs_item_ptr(leaf, slot,
6395                                                   struct btrfs_dev_item);
6396                         ret = read_one_dev(root, leaf, dev_item);
6397                         if (ret)
6398                                 goto error;
6399                 } else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
6400                         struct btrfs_chunk *chunk;
6401                         chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
6402                         ret = read_one_chunk(root, &found_key, leaf, chunk);
6403                         if (ret)
6404                                 goto error;
6405                 }
6406                 path->slots[0]++;
6407         }
6408         ret = 0;
6409 error:
6410         unlock_chunks(root);
6411         mutex_unlock(&uuid_mutex);
6412
6413         btrfs_free_path(path);
6414         return ret;
6415 }
6416
6417 void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
6418 {
6419         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6420         struct btrfs_device *device;
6421
6422         while (fs_devices) {
6423                 mutex_lock(&fs_devices->device_list_mutex);
6424                 list_for_each_entry(device, &fs_devices->devices, dev_list)
6425                         device->dev_root = fs_info->dev_root;
6426                 mutex_unlock(&fs_devices->device_list_mutex);
6427
6428                 fs_devices = fs_devices->seed;
6429         }
6430 }
6431
6432 static void __btrfs_reset_dev_stats(struct btrfs_device *dev)
6433 {
6434         int i;
6435
6436         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
6437                 btrfs_dev_stat_reset(dev, i);
6438 }
6439
6440 int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
6441 {
6442         struct btrfs_key key;
6443         struct btrfs_key found_key;
6444         struct btrfs_root *dev_root = fs_info->dev_root;
6445         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6446         struct extent_buffer *eb;
6447         int slot;
6448         int ret = 0;
6449         struct btrfs_device *device;
6450         struct btrfs_path *path = NULL;
6451         int i;
6452
6453         path = btrfs_alloc_path();
6454         if (!path) {
6455                 ret = -ENOMEM;
6456                 goto out;
6457         }
6458
6459         mutex_lock(&fs_devices->device_list_mutex);
6460         list_for_each_entry(device, &fs_devices->devices, dev_list) {
6461                 int item_size;
6462                 struct btrfs_dev_stats_item *ptr;
6463
6464                 key.objectid = 0;
6465                 key.type = BTRFS_DEV_STATS_KEY;
6466                 key.offset = device->devid;
6467                 ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
6468                 if (ret) {
6469                         __btrfs_reset_dev_stats(device);
6470                         device->dev_stats_valid = 1;
6471                         btrfs_release_path(path);
6472                         continue;
6473                 }
6474                 slot = path->slots[0];
6475                 eb = path->nodes[0];
6476                 btrfs_item_key_to_cpu(eb, &found_key, slot);
6477                 item_size = btrfs_item_size_nr(eb, slot);
6478
6479                 ptr = btrfs_item_ptr(eb, slot,
6480                                      struct btrfs_dev_stats_item);
6481
6482                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
6483                         if (item_size >= (1 + i) * sizeof(__le64))
6484                                 btrfs_dev_stat_set(device, i,
6485                                         btrfs_dev_stats_value(eb, ptr, i));
6486                         else
6487                                 btrfs_dev_stat_reset(device, i);
6488                 }
6489
6490                 device->dev_stats_valid = 1;
6491                 btrfs_dev_stat_print_on_load(device);
6492                 btrfs_release_path(path);
6493         }
6494         mutex_unlock(&fs_devices->device_list_mutex);
6495
6496 out:
6497         btrfs_free_path(path);
6498         return ret < 0 ? ret : 0;
6499 }
6500
6501 static int update_dev_stat_item(struct btrfs_trans_handle *trans,
6502                                 struct btrfs_root *dev_root,
6503                                 struct btrfs_device *device)
6504 {
6505         struct btrfs_path *path;
6506         struct btrfs_key key;
6507         struct extent_buffer *eb;
6508         struct btrfs_dev_stats_item *ptr;
6509         int ret;
6510         int i;
6511
6512         key.objectid = 0;
6513         key.type = BTRFS_DEV_STATS_KEY;
6514         key.offset = device->devid;
6515
6516         path = btrfs_alloc_path();
6517         BUG_ON(!path);
6518         ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
6519         if (ret < 0) {
6520                 printk_in_rcu(KERN_WARNING "BTRFS: "
6521                         "error %d while searching for dev_stats item for device %s!\n",
6522                               ret, rcu_str_deref(device->name));
6523                 goto out;
6524         }
6525
6526         if (ret == 0 &&
6527             btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
6528                 /* need to delete old one and insert a new one */
6529                 ret = btrfs_del_item(trans, dev_root, path);
6530                 if (ret != 0) {
6531                         printk_in_rcu(KERN_WARNING "BTRFS: "
6532                                 "delete too small dev_stats item for device %s failed %d!\n",
6533                                       rcu_str_deref(device->name), ret);
6534                         goto out;
6535                 }
6536                 ret = 1;
6537         }
6538
6539         if (ret == 1) {
6540                 /* need to insert a new item */
6541                 btrfs_release_path(path);
6542                 ret = btrfs_insert_empty_item(trans, dev_root, path,
6543                                               &key, sizeof(*ptr));
6544                 if (ret < 0) {
6545                         printk_in_rcu(KERN_WARNING "BTRFS: "
6546                                           "insert dev_stats item for device %s failed %d!\n",
6547                                       rcu_str_deref(device->name), ret);
6548                         goto out;
6549                 }
6550         }
6551
6552         eb = path->nodes[0];
6553         ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
6554         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
6555                 btrfs_set_dev_stats_value(eb, ptr, i,
6556                                           btrfs_dev_stat_read(device, i));
6557         btrfs_mark_buffer_dirty(eb);
6558
6559 out:
6560         btrfs_free_path(path);
6561         return ret;
6562 }
6563
6564 /*
6565  * called from commit_transaction. Writes all changed device stats to disk.
6566  */
6567 int btrfs_run_dev_stats(struct btrfs_trans_handle *trans,
6568                         struct btrfs_fs_info *fs_info)
6569 {
6570         struct btrfs_root *dev_root = fs_info->dev_root;
6571         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6572         struct btrfs_device *device;
6573         int stats_cnt;
6574         int ret = 0;
6575
6576         mutex_lock(&fs_devices->device_list_mutex);
6577         list_for_each_entry(device, &fs_devices->devices, dev_list) {
6578                 if (!device->dev_stats_valid || !btrfs_dev_stats_dirty(device))
6579                         continue;
6580
6581                 stats_cnt = atomic_read(&device->dev_stats_ccnt);
6582                 ret = update_dev_stat_item(trans, dev_root, device);
6583                 if (!ret)
6584                         atomic_sub(stats_cnt, &device->dev_stats_ccnt);
6585         }
6586         mutex_unlock(&fs_devices->device_list_mutex);
6587
6588         return ret;
6589 }
6590
6591 void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
6592 {
6593         btrfs_dev_stat_inc(dev, index);
6594         btrfs_dev_stat_print_on_error(dev);
6595 }
6596
6597 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
6598 {
6599         if (!dev->dev_stats_valid)
6600                 return;
6601         printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
6602                            "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u\n",
6603                            rcu_str_deref(dev->name),
6604                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
6605                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
6606                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
6607                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
6608                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
6609 }
6610
6611 static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
6612 {
6613         int i;
6614
6615         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
6616                 if (btrfs_dev_stat_read(dev, i) != 0)
6617                         break;
6618         if (i == BTRFS_DEV_STAT_VALUES_MAX)
6619                 return; /* all values == 0, suppress message */
6620
6621         printk_in_rcu(KERN_INFO "BTRFS: "
6622                    "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u\n",
6623                rcu_str_deref(dev->name),
6624                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
6625                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
6626                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
6627                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
6628                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
6629 }
6630
6631 int btrfs_get_dev_stats(struct btrfs_root *root,
6632                         struct btrfs_ioctl_get_dev_stats *stats)
6633 {
6634         struct btrfs_device *dev;
6635         struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
6636         int i;
6637
6638         mutex_lock(&fs_devices->device_list_mutex);
6639         dev = btrfs_find_device(root->fs_info, stats->devid, NULL, NULL);
6640         mutex_unlock(&fs_devices->device_list_mutex);
6641
6642         if (!dev) {
6643                 btrfs_warn(root->fs_info, "get dev_stats failed, device not found");
6644                 return -ENODEV;
6645         } else if (!dev->dev_stats_valid) {
6646                 btrfs_warn(root->fs_info, "get dev_stats failed, not yet valid");
6647                 return -ENODEV;
6648         } else if (stats->flags & BTRFS_DEV_STATS_RESET) {
6649                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
6650                         if (stats->nr_items > i)
6651                                 stats->values[i] =
6652                                         btrfs_dev_stat_read_and_reset(dev, i);
6653                         else
6654                                 btrfs_dev_stat_reset(dev, i);
6655                 }
6656         } else {
6657                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
6658                         if (stats->nr_items > i)
6659                                 stats->values[i] = btrfs_dev_stat_read(dev, i);
6660         }
6661         if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
6662                 stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
6663         return 0;
6664 }
6665
6666 int btrfs_scratch_superblock(struct btrfs_device *device)
6667 {
6668         struct buffer_head *bh;
6669         struct btrfs_super_block *disk_super;
6670
6671         bh = btrfs_read_dev_super(device->bdev);
6672         if (!bh)
6673                 return -EINVAL;
6674         disk_super = (struct btrfs_super_block *)bh->b_data;
6675
6676         memset(&disk_super->magic, 0, sizeof(disk_super->magic));
6677         set_buffer_dirty(bh);
6678         sync_dirty_buffer(bh);
6679         brelse(bh);
6680
6681         return 0;
6682 }
6683
6684 /*
6685  * Update the size of all devices, which is used for writing out the
6686  * super blocks.
6687  */
6688 void btrfs_update_commit_device_size(struct btrfs_fs_info *fs_info)
6689 {
6690         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6691         struct btrfs_device *curr, *next;
6692
6693         if (list_empty(&fs_devices->resized_devices))
6694                 return;
6695
6696         mutex_lock(&fs_devices->device_list_mutex);
6697         lock_chunks(fs_info->dev_root);
6698         list_for_each_entry_safe(curr, next, &fs_devices->resized_devices,
6699                                  resized_list) {
6700                 list_del_init(&curr->resized_list);
6701                 curr->commit_total_bytes = curr->disk_total_bytes;
6702         }
6703         unlock_chunks(fs_info->dev_root);
6704         mutex_unlock(&fs_devices->device_list_mutex);
6705 }
6706
6707 /* Must be invoked during the transaction commit */
6708 void btrfs_update_commit_device_bytes_used(struct btrfs_root *root,
6709                                         struct btrfs_transaction *transaction)
6710 {
6711         struct extent_map *em;
6712         struct map_lookup *map;
6713         struct btrfs_device *dev;
6714         int i;
6715
6716         if (list_empty(&transaction->pending_chunks))
6717                 return;
6718
6719         /* In order to kick the device replace finish process */
6720         lock_chunks(root);
6721         list_for_each_entry(em, &transaction->pending_chunks, list) {
6722                 map = (struct map_lookup *)em->bdev;
6723
6724                 for (i = 0; i < map->num_stripes; i++) {
6725                         dev = map->stripes[i].dev;
6726                         dev->commit_bytes_used = dev->bytes_used;
6727                 }
6728         }
6729         unlock_chunks(root);
6730 }