drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-path-selector.h"
  11 #include "dm-uevent.h"
  12
  13 #include <linux/ctype.h>
  14 #include <linux/init.h>
  15 #include <linux/mempool.h>
  16 #include <linux/module.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/slab.h>
  19 #include <linux/time.h>
  20 #include <linux/workqueue.h>
  21 #include <scsi/scsi_dh.h>
  22 #include <asm/atomic.h>
  23
  24 #define DM_MSG_PREFIX "multipath"
  25 #define MESG_STR(x) x, sizeof(x)
  26
  27 /* Path properties */
  28 struct pgpath {
  29         struct list_head list;
  30
  31         struct priority_group *pg;      /* Owning PG */
  32         unsigned is_active;             /* Path status */
  33         unsigned fail_count;            /* Cumulative failure count */
  34
  35         struct dm_path path;
  36         struct work_struct deactivate_path;
  37         struct work_struct activate_path;
  38 };
  39
  40 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  41
  42 /*
  43  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  44  * Each has a path selector which controls which path gets used.
  45  */
  46 struct priority_group {
  47         struct list_head list;
  48
  49         struct multipath *m;            /* Owning multipath instance */
  50         struct path_selector ps;
  51
  52         unsigned pg_num;                /* Reference number */
  53         unsigned bypassed;              /* Temporarily bypass this PG? */
  54
  55         unsigned nr_pgpaths;            /* Number of paths in PG */
  56         struct list_head pgpaths;
  57 };
  58
  59 /* Multipath context */
  60 struct multipath {
  61         struct list_head list;
  62         struct dm_target *ti;
  63
  64         spinlock_t lock;
  65
  66         const char *hw_handler_name;
  67         char *hw_handler_params;
  68         unsigned nr_priority_groups;
  69         struct list_head priority_groups;
  70         unsigned pg_init_required;      /* pg_init needs calling? */
  71         unsigned pg_init_in_progress;   /* Only one pg_init allowed at once */
  72
  73         unsigned nr_valid_paths;        /* Total number of usable paths */
  74         struct pgpath *current_pgpath;
  75         struct priority_group *current_pg;
  76         struct priority_group *next_pg; /* Switch to this PG if set */
  77         unsigned repeat_count;          /* I/Os left before calling PS again */
  78
  79         unsigned queue_io;              /* Must we queue all I/O? */
  80         unsigned queue_if_no_path;      /* Queue I/O if last path fails? */
  81         unsigned saved_queue_if_no_path;/* Saved state during suspension */
  82         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  83         unsigned pg_init_count;         /* Number of times pg_init called */
  84
  85         struct work_struct process_queued_ios;
  86         struct list_head queued_ios;
  87         unsigned queue_size;
  88
  89         struct work_struct trigger_event;
  90
  91         /*
  92          * We must use a mempool of dm_mpath_io structs so that we
  93          * can resubmit bios on error.
  94          */
  95         mempool_t *mpio_pool;
  96 };
  97
  98 /*
  99  * Context information attached to each bio we process.
 100  */
 101 struct dm_mpath_io {
 102         struct pgpath *pgpath;
 103         size_t nr_bytes;
 104 };
 105
 106 typedef int (*action_fn) (struct pgpath *pgpath);
 107
 108 #define MIN_IOS 256     /* Mempool size */
 109
 110 static struct kmem_cache *_mpio_cache;
 111
 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 113 static void process_queued_ios(struct work_struct *work);
 114 static void trigger_event(struct work_struct *work);
 115 static void activate_path(struct work_struct *work);
 116 static void deactivate_path(struct work_struct *work);
 117
 118
 119 /*-----------------------------------------------
 120  * Allocation routines
 121  *-----------------------------------------------*/
 122
 123 static struct pgpath *alloc_pgpath(void)
 124 {
 125         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 126
 127         if (pgpath) {
 128                 pgpath->is_active = 1;
 129                 INIT_WORK(&pgpath->deactivate_path, deactivate_path);
 130                 INIT_WORK(&pgpath->activate_path, activate_path);
 131         }
 132
 133         return pgpath;
 134 }
 135
 136 static void free_pgpath(struct pgpath *pgpath)
 137 {
 138         kfree(pgpath);
 139 }
 140
 141 static void deactivate_path(struct work_struct *work)
 142 {
 143         struct pgpath *pgpath =
 144                 container_of(work, struct pgpath, deactivate_path);
 145
 146         blk_abort_queue(pgpath->path.dev->bdev->bd_disk->queue);
 147 }
 148
 149 static struct priority_group *alloc_priority_group(void)
 150 {
 151         struct priority_group *pg;
 152
 153         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 154
 155         if (pg)
 156                 INIT_LIST_HEAD(&pg->pgpaths);
 157
 158         return pg;
 159 }
 160
 161 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 162 {
 163         struct pgpath *pgpath, *tmp;
 164         struct multipath *m = ti->private;
 165
 166         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 167                 list_del(&pgpath->list);
 168                 if (m->hw_handler_name)
 169                         scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
 170                 dm_put_device(ti, pgpath->path.dev);
 171                 free_pgpath(pgpath);
 172         }
 173 }
 174
 175 static void free_priority_group(struct priority_group *pg,
 176                                 struct dm_target *ti)
 177 {
 178         struct path_selector *ps = &pg->ps;
 179
 180         if (ps->type) {
 181                 ps->type->destroy(ps);
 182                 dm_put_path_selector(ps->type);
 183         }
 184
 185         free_pgpaths(&pg->pgpaths, ti);
 186         kfree(pg);
 187 }
 188
 189 static struct multipath *alloc_multipath(struct dm_target *ti)
 190 {
 191         struct multipath *m;
 192
 193         m = kzalloc(sizeof(*m), GFP_KERNEL);
 194         if (m) {
 195                 INIT_LIST_HEAD(&m->priority_groups);
 196                 INIT_LIST_HEAD(&m->queued_ios);
 197                 spin_lock_init(&m->lock);
 198                 m->queue_io = 1;
 199                 INIT_WORK(&m->process_queued_ios, process_queued_ios);
 200                 INIT_WORK(&m->trigger_event, trigger_event);
 201                 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
 202                 if (!m->mpio_pool) {
 203                         kfree(m);
 204                         return NULL;
 205                 }
 206                 m->ti = ti;
 207                 ti->private = m;
 208         }
 209
 210         return m;
 211 }
 212
 213 static void free_multipath(struct multipath *m)
 214 {
 215         struct priority_group *pg, *tmp;
 216
 217         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 218                 list_del(&pg->list);
 219                 free_priority_group(pg, m->ti);
 220         }
 221
 222         kfree(m->hw_handler_name);
 223         kfree(m->hw_handler_params);
 224         mempool_destroy(m->mpio_pool);
 225         kfree(m);
 226 }
 227
 228
 229 /*-----------------------------------------------
 230  * Path selection
 231  *-----------------------------------------------*/
 232
 233 static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
 234 {
 235         m->current_pg = pgpath->pg;
 236
 237         /* Must we initialise the PG first, and queue I/O till it's ready? */
 238         if (m->hw_handler_name) {
 239                 m->pg_init_required = 1;
 240                 m->queue_io = 1;
 241         } else {
 242                 m->pg_init_required = 0;
 243                 m->queue_io = 0;
 244         }
 245
 246         m->pg_init_count = 0;
 247 }
 248
 249 static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
 250                                size_t nr_bytes)
 251 {
 252         struct dm_path *path;
 253
 254         path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
 255         if (!path)
 256                 return -ENXIO;
 257
 258         m->current_pgpath = path_to_pgpath(path);
 259
 260         if (m->current_pg != pg)
 261                 __switch_pg(m, m->current_pgpath);
 262
 263         return 0;
 264 }
 265
 266 static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
 267 {
 268         struct priority_group *pg;
 269         unsigned bypassed = 1;
 270
 271         if (!m->nr_valid_paths)
 272                 goto failed;
 273
 274         /* Were we instructed to switch PG? */
 275         if (m->next_pg) {
 276                 pg = m->next_pg;
 277                 m->next_pg = NULL;
 278                 if (!__choose_path_in_pg(m, pg, nr_bytes))
 279                         return;
 280         }
 281
 282         /* Don't change PG until it has no remaining paths */
 283         if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
 284                 return;
 285
 286         /*
 287          * Loop through priority groups until we find a valid path.
 288          * First time we skip PGs marked 'bypassed'.
 289          * Second time we only try the ones we skipped.
 290          */
 291         do {
 292                 list_for_each_entry(pg, &m->priority_groups, list) {
 293                         if (pg->bypassed == bypassed)
 294                                 continue;
 295                         if (!__choose_path_in_pg(m, pg, nr_bytes))
 296                                 return;
 297                 }
 298         } while (bypassed--);
 299
 300 failed:
 301         m->current_pgpath = NULL;
 302         m->current_pg = NULL;
 303 }
 304
 305 /*
 306  * Check whether bios must be queued in the device-mapper core rather
 307  * than here in the target.
 308  *
 309  * m->lock must be held on entry.
 310  *
 311  * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
 312  * same value then we are not between multipath_presuspend()
 313  * and multipath_resume() calls and we have no need to check
 314  * for the DMF_NOFLUSH_SUSPENDING flag.
 315  */
 316 static int __must_push_back(struct multipath *m)
 317 {
 318         return (m->queue_if_no_path != m->saved_queue_if_no_path &&
 319                 dm_noflush_suspending(m->ti));
 320 }
 321
 322 static int map_io(struct multipath *m, struct request *clone,
 323                   struct dm_mpath_io *mpio, unsigned was_queued)
 324 {
 325         int r = DM_MAPIO_REMAPPED;
 326         size_t nr_bytes = blk_rq_bytes(clone);
 327         unsigned long flags;
 328         struct pgpath *pgpath;
 329         struct block_device *bdev;
 330
 331         spin_lock_irqsave(&m->lock, flags);
 332
 333         /* Do we need to select a new pgpath? */
 334         if (!m->current_pgpath ||
 335             (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
 336                 __choose_pgpath(m, nr_bytes);
 337
 338         pgpath = m->current_pgpath;
 339
 340         if (was_queued)
 341                 m->queue_size--;
 342
 343         if ((pgpath && m->queue_io) ||
 344             (!pgpath && m->queue_if_no_path)) {
 345                 /* Queue for the daemon to resubmit */
 346                 list_add_tail(&clone->queuelist, &m->queued_ios);
 347                 m->queue_size++;
 348                 if ((m->pg_init_required && !m->pg_init_in_progress) ||
 349                     !m->queue_io)
 350                         queue_work(kmultipathd, &m->process_queued_ios);
 351                 pgpath = NULL;
 352                 r = DM_MAPIO_SUBMITTED;
 353         } else if (pgpath) {
 354                 bdev = pgpath->path.dev->bdev;
 355                 clone->q = bdev_get_queue(bdev);
 356                 clone->rq_disk = bdev->bd_disk;
 357         } else if (__must_push_back(m))
 358                 r = DM_MAPIO_REQUEUE;
 359         else
 360                 r = -EIO;       /* Failed */
 361
 362         mpio->pgpath = pgpath;
 363         mpio->nr_bytes = nr_bytes;
 364
 365         if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
 366                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
 367                                               nr_bytes);
 368
 369         spin_unlock_irqrestore(&m->lock, flags);
 370
 371         return r;
 372 }
 373
 374 /*
 375  * If we run out of usable paths, should we queue I/O or error it?
 376  */
 377 static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
 378                             unsigned save_old_value)
 379 {
 380         unsigned long flags;
 381
 382         spin_lock_irqsave(&m->lock, flags);
 383
 384         if (save_old_value)
 385                 m->saved_queue_if_no_path = m->queue_if_no_path;
 386         else
 387                 m->saved_queue_if_no_path = queue_if_no_path;
 388         m->queue_if_no_path = queue_if_no_path;
 389         if (!m->queue_if_no_path && m->queue_size)
 390                 queue_work(kmultipathd, &m->process_queued_ios);
 391
 392         spin_unlock_irqrestore(&m->lock, flags);
 393
 394         return 0;
 395 }
 396
 397 /*-----------------------------------------------------------------
 398  * The multipath daemon is responsible for resubmitting queued ios.
 399  *---------------------------------------------------------------*/
 400
 401 static void dispatch_queued_ios(struct multipath *m)
 402 {
 403         int r;
 404         unsigned long flags;
 405         struct dm_mpath_io *mpio;
 406         union map_info *info;
 407         struct request *clone, *n;
 408         LIST_HEAD(cl);
 409
 410         spin_lock_irqsave(&m->lock, flags);
 411         list_splice_init(&m->queued_ios, &cl);
 412         spin_unlock_irqrestore(&m->lock, flags);
 413
 414         list_for_each_entry_safe(clone, n, &cl, queuelist) {
 415                 list_del_init(&clone->queuelist);
 416
 417                 info = dm_get_rq_mapinfo(clone);
 418                 mpio = info->ptr;
 419
 420                 r = map_io(m, clone, mpio, 1);
 421                 if (r < 0) {
 422                         mempool_free(mpio, m->mpio_pool);
 423                         dm_kill_unmapped_request(clone, r);
 424                 } else if (r == DM_MAPIO_REMAPPED)
 425                         dm_dispatch_request(clone);
 426                 else if (r == DM_MAPIO_REQUEUE) {
 427                         mempool_free(mpio, m->mpio_pool);
 428                         dm_requeue_unmapped_request(clone);
 429                 }
 430         }
 431 }
 432
 433 static void process_queued_ios(struct work_struct *work)
 434 {
 435         struct multipath *m =
 436                 container_of(work, struct multipath, process_queued_ios);
 437         struct pgpath *pgpath = NULL, *tmp;
 438         unsigned must_queue = 1;
 439         unsigned long flags;
 440
 441         spin_lock_irqsave(&m->lock, flags);
 442
 443         if (!m->queue_size)
 444                 goto out;
 445
 446         if (!m->current_pgpath)
 447                 __choose_pgpath(m, 0);
 448
 449         pgpath = m->current_pgpath;
 450
 451         if ((pgpath && !m->queue_io) ||
 452             (!pgpath && !m->queue_if_no_path))
 453                 must_queue = 0;
 454
 455         if (m->pg_init_required && !m->pg_init_in_progress && pgpath) {
 456                 m->pg_init_count++;
 457                 m->pg_init_required = 0;
 458                 list_for_each_entry(tmp, &pgpath->pg->pgpaths, list) {
 459                         if (queue_work(kmpath_handlerd, &tmp->activate_path))
 460                                 m->pg_init_in_progress++;
 461                 }
 462         }
 463 out:
 464         spin_unlock_irqrestore(&m->lock, flags);
 465         if (!must_queue)
 466                 dispatch_queued_ios(m);
 467 }
 468
 469 /*
 470  * An event is triggered whenever a path is taken out of use.
 471  * Includes path failure and PG bypass.
 472  */
 473 static void trigger_event(struct work_struct *work)
 474 {
 475         struct multipath *m =
 476                 container_of(work, struct multipath, trigger_event);
 477
 478         dm_table_event(m->ti->table);
 479 }
 480
 481 /*-----------------------------------------------------------------
 482  * Constructor/argument parsing:
 483  * <#multipath feature args> [<arg>]*
 484  * <#hw_handler args> [hw_handler [<arg>]*]
 485  * <#priority groups>
 486  * <initial priority group>
 487  *     [<selector> <#selector args> [<arg>]*
 488  *      <#paths> <#per-path selector args>
 489  *         [<path> [<arg>]* ]+ ]+
 490  *---------------------------------------------------------------*/
 491 struct param {
 492         unsigned min;
 493         unsigned max;
 494         char *error;
 495 };
 496
 497 static int read_param(struct param *param, char *str, unsigned *v, char **error)
 498 {
 499         if (!str ||
 500             (sscanf(str, "%u", v) != 1) ||
 501             (*v < param->min) ||
 502             (*v > param->max)) {
 503                 *error = param->error;
 504                 return -EINVAL;
 505         }
 506
 507         return 0;
 508 }
 509
 510 struct arg_set {
 511         unsigned argc;
 512         char **argv;
 513 };
 514
 515 static char *shift(struct arg_set *as)
 516 {
 517         char *r;
 518
 519         if (as->argc) {
 520                 as->argc--;
 521                 r = *as->argv;
 522                 as->argv++;
 523                 return r;
 524         }
 525
 526         return NULL;
 527 }
 528
 529 static void consume(struct arg_set *as, unsigned n)
 530 {
 531         BUG_ON (as->argc < n);
 532         as->argc -= n;
 533         as->argv += n;
 534 }
 535
 536 static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
 537                                struct dm_target *ti)
 538 {
 539         int r;
 540         struct path_selector_type *pst;
 541         unsigned ps_argc;
 542
 543         static struct param _params[] = {
 544                 {0, 1024, "invalid number of path selector args"},
 545         };
 546
 547         pst = dm_get_path_selector(shift(as));
 548         if (!pst) {
 549                 ti->error = "unknown path selector type";
 550                 return -EINVAL;
 551         }
 552
 553         r = read_param(_params, shift(as), &ps_argc, &ti->error);
 554         if (r) {
 555                 dm_put_path_selector(pst);
 556                 return -EINVAL;
 557         }
 558
 559         if (ps_argc > as->argc) {
 560                 dm_put_path_selector(pst);
 561                 ti->error = "not enough arguments for path selector";
 562                 return -EINVAL;
 563         }
 564
 565         r = pst->create(&pg->ps, ps_argc, as->argv);
 566         if (r) {
 567                 dm_put_path_selector(pst);
 568                 ti->error = "path selector constructor failed";
 569                 return r;
 570         }
 571
 572         pg->ps.type = pst;
 573         consume(as, ps_argc);
 574
 575         return 0;
 576 }
 577
 578 static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
 579                                struct dm_target *ti)
 580 {
 581         int r;
 582         struct pgpath *p;
 583         struct multipath *m = ti->private;
 584
 585         /* we need at least a path arg */
 586         if (as->argc < 1) {
 587                 ti->error = "no device given";
 588                 return ERR_PTR(-EINVAL);
 589         }
 590
 591         p = alloc_pgpath();
 592         if (!p)
 593                 return ERR_PTR(-ENOMEM);
 594
 595         r = dm_get_device(ti, shift(as), ti->begin, ti->len,
 596                           dm_table_get_mode(ti->table), &p->path.dev);
 597         if (r) {
 598                 ti->error = "error getting device";
 599                 goto bad;
 600         }
 601
 602         if (m->hw_handler_name) {
 603                 struct request_queue *q = bdev_get_queue(p->path.dev->bdev);
 604
 605                 r = scsi_dh_attach(q, m->hw_handler_name);
 606                 if (r == -EBUSY) {
 607                         /*
 608                          * Already attached to different hw_handler,
 609                          * try to reattach with correct one.
 610                          */
 611                         scsi_dh_detach(q);
 612                         r = scsi_dh_attach(q, m->hw_handler_name);
 613                 }
 614
 615                 if (r < 0) {
 616                         ti->error = "error attaching hardware handler";
 617                         dm_put_device(ti, p->path.dev);
 618                         goto bad;
 619                 }
 620
 621                 if (m->hw_handler_params) {
 622                         r = scsi_dh_set_params(q, m->hw_handler_params);
 623                         if (r < 0) {
 624                                 ti->error = "unable to set hardware "
 625                                                         "handler parameters";
 626                                 scsi_dh_detach(q);
 627                                 dm_put_device(ti, p->path.dev);
 628                                 goto bad;
 629                         }
 630                 }
 631         }
 632
 633         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 634         if (r) {
 635                 dm_put_device(ti, p->path.dev);
 636                 goto bad;
 637         }
 638
 639         return p;
 640
 641  bad:
 642         free_pgpath(p);
 643         return ERR_PTR(r);
 644 }
 645
 646 static struct priority_group *parse_priority_group(struct arg_set *as,
 647                                                    struct multipath *m)
 648 {
 649         static struct param _params[] = {
 650                 {1, 1024, "invalid number of paths"},
 651                 {0, 1024, "invalid number of selector args"}
 652         };
 653
 654         int r;
 655         unsigned i, nr_selector_args, nr_params;
 656         struct priority_group *pg;
 657         struct dm_target *ti = m->ti;
 658
 659         if (as->argc < 2) {
 660                 as->argc = 0;
 661                 ti->error = "not enough priority group arguments";
 662                 return ERR_PTR(-EINVAL);
 663         }
 664
 665         pg = alloc_priority_group();
 666         if (!pg) {
 667                 ti->error = "couldn't allocate priority group";
 668                 return ERR_PTR(-ENOMEM);
 669         }
 670         pg->m = m;
 671
 672         r = parse_path_selector(as, pg, ti);
 673         if (r)
 674                 goto bad;
 675
 676         /*
 677          * read the paths
 678          */
 679         r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
 680         if (r)
 681                 goto bad;
 682
 683         r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
 684         if (r)
 685                 goto bad;
 686
 687         nr_params = 1 + nr_selector_args;
 688         for (i = 0; i < pg->nr_pgpaths; i++) {
 689                 struct pgpath *pgpath;
 690                 struct arg_set path_args;
 691
 692                 if (as->argc < nr_params) {
 693                         ti->error = "not enough path parameters";
 694                         goto bad;
 695                 }
 696
 697                 path_args.argc = nr_params;
 698                 path_args.argv = as->argv;
 699
 700                 pgpath = parse_path(&path_args, &pg->ps, ti);
 701                 if (IS_ERR(pgpath)) {
 702                         r = PTR_ERR(pgpath);
 703                         goto bad;
 704                 }
 705
 706                 pgpath->pg = pg;
 707                 list_add_tail(&pgpath->list, &pg->pgpaths);
 708                 consume(as, nr_params);
 709         }
 710
 711         return pg;
 712
 713  bad:
 714         free_priority_group(pg, ti);
 715         return ERR_PTR(r);
 716 }
 717
 718 static int parse_hw_handler(struct arg_set *as, struct multipath *m)
 719 {
 720         unsigned hw_argc;
 721         int ret;
 722         struct dm_target *ti = m->ti;
 723
 724         static struct param _params[] = {
 725                 {0, 1024, "invalid number of hardware handler args"},
 726         };
 727
 728         if (read_param(_params, shift(as), &hw_argc, &ti->error))
 729                 return -EINVAL;
 730
 731         if (!hw_argc)
 732                 return 0;
 733
 734         if (hw_argc > as->argc) {
 735                 ti->error = "not enough arguments for hardware handler";
 736                 return -EINVAL;
 737         }
 738
 739         m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL);
 740         request_module("scsi_dh_%s", m->hw_handler_name);
 741         if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
 742                 ti->error = "unknown hardware handler type";
 743                 ret = -EINVAL;
 744                 goto fail;
 745         }
 746
 747         if (hw_argc > 1) {
 748                 char *p;
 749                 int i, j, len = 4;
 750
 751                 for (i = 0; i <= hw_argc - 2; i++)
 752                         len += strlen(as->argv[i]) + 1;
 753                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
 754                 if (!p) {
 755                         ti->error = "memory allocation failed";
 756                         ret = -ENOMEM;
 757                         goto fail;
 758                 }
 759                 j = sprintf(p, "%d", hw_argc - 1);
 760                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
 761                         j = sprintf(p, "%s", as->argv[i]);
 762         }
 763         consume(as, hw_argc - 1);
 764
 765         return 0;
 766 fail:
 767         kfree(m->hw_handler_name);
 768         m->hw_handler_name = NULL;
 769         return ret;
 770 }
 771
 772 static int parse_features(struct arg_set *as, struct multipath *m)
 773 {
 774         int r;
 775         unsigned argc;
 776         struct dm_target *ti = m->ti;
 777         const char *param_name;
 778
 779         static struct param _params[] = {
 780                 {0, 3, "invalid number of feature args"},
 781                 {1, 50, "pg_init_retries must be between 1 and 50"},
 782         };
 783
 784         r = read_param(_params, shift(as), &argc, &ti->error);
 785         if (r)
 786                 return -EINVAL;
 787
 788         if (!argc)
 789                 return 0;
 790
 791         do {
 792                 param_name = shift(as);
 793                 argc--;
 794
 795                 if (!strnicmp(param_name, MESG_STR("queue_if_no_path"))) {
 796                         r = queue_if_no_path(m, 1, 0);
 797                         continue;
 798                 }
 799
 800                 if (!strnicmp(param_name, MESG_STR("pg_init_retries")) &&
 801                     (argc >= 1)) {
 802                         r = read_param(_params + 1, shift(as),
 803                                        &m->pg_init_retries, &ti->error);
 804                         argc--;
 805                         continue;
 806                 }
 807
 808                 ti->error = "Unrecognised multipath feature request";
 809                 r = -EINVAL;
 810         } while (argc && !r);
 811
 812         return r;
 813 }
 814
 815 static int multipath_ctr(struct dm_target *ti, unsigned int argc,
 816                          char **argv)
 817 {
 818         /* target parameters */
 819         static struct param _params[] = {
 820                 {1, 1024, "invalid number of priority groups"},
 821                 {1, 1024, "invalid initial priority group number"},
 822         };
 823
 824         int r;
 825         struct multipath *m;
 826         struct arg_set as;
 827         unsigned pg_count = 0;
 828         unsigned next_pg_num;
 829
 830         as.argc = argc;
 831         as.argv = argv;
 832
 833         m = alloc_multipath(ti);
 834         if (!m) {
 835                 ti->error = "can't allocate multipath";
 836                 return -EINVAL;
 837         }
 838
 839         r = parse_features(&as, m);
 840         if (r)
 841                 goto bad;
 842
 843         r = parse_hw_handler(&as, m);
 844         if (r)
 845                 goto bad;
 846
 847         r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
 848         if (r)
 849                 goto bad;
 850
 851         r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
 852         if (r)
 853                 goto bad;
 854
 855         /* parse the priority groups */
 856         while (as.argc) {
 857                 struct priority_group *pg;
 858
 859                 pg = parse_priority_group(&as, m);
 860                 if (IS_ERR(pg)) {
 861                         r = PTR_ERR(pg);
 862                         goto bad;
 863                 }
 864
 865                 m->nr_valid_paths += pg->nr_pgpaths;
 866                 list_add_tail(&pg->list, &m->priority_groups);
 867                 pg_count++;
 868                 pg->pg_num = pg_count;
 869                 if (!--next_pg_num)
 870                         m->next_pg = pg;
 871         }
 872
 873         if (pg_count != m->nr_priority_groups) {
 874                 ti->error = "priority group count mismatch";
 875                 r = -EINVAL;
 876                 goto bad;
 877         }
 878
 879         ti->num_flush_requests = 1;
 880
 881         return 0;
 882
 883  bad:
 884         free_multipath(m);
 885         return r;
 886 }
 887
 888 static void multipath_dtr(struct dm_target *ti)
 889 {
 890         struct multipath *m = (struct multipath *) ti->private;
 891
 892         flush_workqueue(kmpath_handlerd);
 893         flush_workqueue(kmultipathd);
 894         flush_scheduled_work();
 895         free_multipath(m);
 896 }
 897
 898 /*
 899  * Map cloned requests
 900  */
 901 static int multipath_map(struct dm_target *ti, struct request *clone,
 902                          union map_info *map_context)
 903 {
 904         int r;
 905         struct dm_mpath_io *mpio;
 906         struct multipath *m = (struct multipath *) ti->private;
 907
 908         mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
 909         if (!mpio)
 910                 /* ENOMEM, requeue */
 911                 return DM_MAPIO_REQUEUE;
 912         memset(mpio, 0, sizeof(*mpio));
 913
 914         map_context->ptr = mpio;
 915         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 916         r = map_io(m, clone, mpio, 0);
 917         if (r < 0 || r == DM_MAPIO_REQUEUE)
 918                 mempool_free(mpio, m->mpio_pool);
 919
 920         return r;
 921 }
 922
 923 /*
 924  * Take a path out of use.
 925  */
 926 static int fail_path(struct pgpath *pgpath)
 927 {
 928         unsigned long flags;
 929         struct multipath *m = pgpath->pg->m;
 930
 931         spin_lock_irqsave(&m->lock, flags);
 932
 933         if (!pgpath->is_active)
 934                 goto out;
 935
 936         DMWARN("Failing path %s.", pgpath->path.dev->name);
 937
 938         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
 939         pgpath->is_active = 0;
 940         pgpath->fail_count++;
 941
 942         m->nr_valid_paths--;
 943
 944         if (pgpath == m->current_pgpath)
 945                 m->current_pgpath = NULL;
 946
 947         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
 948                       pgpath->path.dev->name, m->nr_valid_paths);
 949
 950         schedule_work(&m->trigger_event);
 951         queue_work(kmultipathd, &pgpath->deactivate_path);
 952
 953 out:
 954         spin_unlock_irqrestore(&m->lock, flags);
 955
 956         return 0;
 957 }
 958
 959 /*
 960  * Reinstate a previously-failed path
 961  */
 962 static int reinstate_path(struct pgpath *pgpath)
 963 {
 964         int r = 0;
 965         unsigned long flags;
 966         struct multipath *m = pgpath->pg->m;
 967
 968         spin_lock_irqsave(&m->lock, flags);
 969
 970         if (pgpath->is_active)
 971                 goto out;
 972
 973         if (!pgpath->pg->ps.type->reinstate_path) {
 974                 DMWARN("Reinstate path not supported by path selector %s",
 975                        pgpath->pg->ps.type->name);
 976                 r = -EINVAL;
 977                 goto out;
 978         }
 979
 980         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
 981         if (r)
 982                 goto out;
 983
 984         pgpath->is_active = 1;
 985
 986         if (!m->nr_valid_paths++ && m->queue_size) {
 987                 m->current_pgpath = NULL;
 988                 queue_work(kmultipathd, &m->process_queued_ios);
 989         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
 990                 if (queue_work(kmpath_handlerd, &pgpath->activate_path))
 991                         m->pg_init_in_progress++;
 992         }
 993
 994         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
 995                       pgpath->path.dev->name, m->nr_valid_paths);
 996
 997         schedule_work(&m->trigger_event);
 998
 999 out:
1000         spin_unlock_irqrestore(&m->lock, flags);
1001
1002         return r;
1003 }
1004
1005 /*
1006  * Fail or reinstate all paths that match the provided struct dm_dev.
1007  */
1008 static int action_dev(struct multipath *m, struct dm_dev *dev,
1009                       action_fn action)
1010 {
1011         int r = 0;
1012         struct pgpath *pgpath;
1013         struct priority_group *pg;
1014
1015         list_for_each_entry(pg, &m->priority_groups, list) {
1016                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1017                         if (pgpath->path.dev == dev)
1018                                 r = action(pgpath);
1019                 }
1020         }
1021
1022         return r;
1023 }
1024
1025 /*
1026  * Temporarily try to avoid having to use the specified PG
1027  */
1028 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1029                       int bypassed)
1030 {
1031         unsigned long flags;
1032
1033         spin_lock_irqsave(&m->lock, flags);
1034
1035         pg->bypassed = bypassed;
1036         m->current_pgpath = NULL;
1037         m->current_pg = NULL;
1038
1039         spin_unlock_irqrestore(&m->lock, flags);
1040
1041         schedule_work(&m->trigger_event);
1042 }
1043
1044 /*
1045  * Switch to using the specified PG from the next I/O that gets mapped
1046  */
1047 static int switch_pg_num(struct multipath *m, const char *pgstr)
1048 {
1049         struct priority_group *pg;
1050         unsigned pgnum;
1051         unsigned long flags;
1052
1053         if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1054             (pgnum > m->nr_priority_groups)) {
1055                 DMWARN("invalid PG number supplied to switch_pg_num");
1056                 return -EINVAL;
1057         }
1058
1059         spin_lock_irqsave(&m->lock, flags);
1060         list_for_each_entry(pg, &m->priority_groups, list) {
1061                 pg->bypassed = 0;
1062                 if (--pgnum)
1063                         continue;
1064
1065                 m->current_pgpath = NULL;
1066                 m->current_pg = NULL;
1067                 m->next_pg = pg;
1068         }
1069         spin_unlock_irqrestore(&m->lock, flags);
1070
1071         schedule_work(&m->trigger_event);
1072         return 0;
1073 }
1074
1075 /*
1076  * Set/clear bypassed status of a PG.
1077  * PGs are numbered upwards from 1 in the order they were declared.
1078  */
1079 static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1080 {
1081         struct priority_group *pg;
1082         unsigned pgnum;
1083
1084         if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1085             (pgnum > m->nr_priority_groups)) {
1086                 DMWARN("invalid PG number supplied to bypass_pg");
1087                 return -EINVAL;
1088         }
1089
1090         list_for_each_entry(pg, &m->priority_groups, list) {
1091                 if (!--pgnum)
1092                         break;
1093         }
1094
1095         bypass_pg(m, pg, bypassed);
1096         return 0;
1097 }
1098
1099 /*
1100  * Should we retry pg_init immediately?
1101  */
1102 static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1103 {
1104         unsigned long flags;
1105         int limit_reached = 0;
1106
1107         spin_lock_irqsave(&m->lock, flags);
1108
1109         if (m->pg_init_count <= m->pg_init_retries)
1110                 m->pg_init_required = 1;
1111         else
1112                 limit_reached = 1;
1113
1114         spin_unlock_irqrestore(&m->lock, flags);
1115
1116         return limit_reached;
1117 }
1118
1119 static void pg_init_done(void *data, int errors)
1120 {
1121         struct dm_path *path = data;
1122         struct pgpath *pgpath = path_to_pgpath(path);
1123         struct priority_group *pg = pgpath->pg;
1124         struct multipath *m = pg->m;
1125         unsigned long flags;
1126
1127         /* device or driver problems */
1128         switch (errors) {
1129         case SCSI_DH_OK:
1130                 break;
1131         case SCSI_DH_NOSYS:
1132                 if (!m->hw_handler_name) {
1133                         errors = 0;
1134                         break;
1135                 }
1136                 DMERR("Cannot failover device because scsi_dh_%s was not "
1137                       "loaded.", m->hw_handler_name);
1138                 /*
1139                  * Fail path for now, so we do not ping pong
1140                  */
1141                 fail_path(pgpath);
1142                 break;
1143         case SCSI_DH_DEV_TEMP_BUSY:
1144                 /*
1145                  * Probably doing something like FW upgrade on the
1146                  * controller so try the other pg.
1147                  */
1148                 bypass_pg(m, pg, 1);
1149                 break;
1150         /* TODO: For SCSI_DH_RETRY we should wait a couple seconds */
1151         case SCSI_DH_RETRY:
1152         case SCSI_DH_IMM_RETRY:
1153         case SCSI_DH_RES_TEMP_UNAVAIL:
1154                 if (pg_init_limit_reached(m, pgpath))
1155                         fail_path(pgpath);
1156                 errors = 0;
1157                 break;
1158         default:
1159                 /*
1160                  * We probably do not want to fail the path for a device
1161                  * error, but this is what the old dm did. In future
1162                  * patches we can do more advanced handling.
1163                  */
1164                 fail_path(pgpath);
1165         }
1166
1167         spin_lock_irqsave(&m->lock, flags);
1168         if (errors) {
1169                 if (pgpath == m->current_pgpath) {
1170                         DMERR("Could not failover device. Error %d.", errors);
1171                         m->current_pgpath = NULL;
1172                         m->current_pg = NULL;
1173                 }
1174         } else if (!m->pg_init_required) {
1175                 m->queue_io = 0;
1176                 pg->bypassed = 0;
1177         }
1178
1179         m->pg_init_in_progress--;
1180         if (!m->pg_init_in_progress)
1181                 queue_work(kmultipathd, &m->process_queued_ios);
1182         spin_unlock_irqrestore(&m->lock, flags);
1183 }
1184
1185 static void activate_path(struct work_struct *work)
1186 {
1187         struct pgpath *pgpath =
1188                 container_of(work, struct pgpath, activate_path);
1189
1190         scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1191                                 pg_init_done, &pgpath->path);
1192 }
1193
1194 /*
1195  * end_io handling
1196  */
1197 static int do_end_io(struct multipath *m, struct request *clone,
1198                      int error, struct dm_mpath_io *mpio)
1199 {
1200         /*
1201          * We don't queue any clone request inside the multipath target
1202          * during end I/O handling, since those clone requests don't have
1203          * bio clones.  If we queue them inside the multipath target,
1204          * we need to make bio clones, that requires memory allocation.
1205          * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1206          *  don't have bio clones.)
1207          * Instead of queueing the clone request here, we queue the original
1208          * request into dm core, which will remake a clone request and
1209          * clone bios for it and resubmit it later.
1210          */
1211         int r = DM_ENDIO_REQUEUE;
1212         unsigned long flags;
1213
1214         if (!error && !clone->errors)
1215                 return 0;       /* I/O complete */
1216
1217         if (error == -EOPNOTSUPP)
1218                 return error;
1219
1220         if (mpio->pgpath)
1221                 fail_path(mpio->pgpath);
1222
1223         spin_lock_irqsave(&m->lock, flags);
1224         if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
1225                 r = -EIO;
1226         spin_unlock_irqrestore(&m->lock, flags);
1227
1228         return r;
1229 }
1230
1231 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1232                             int error, union map_info *map_context)
1233 {
1234         struct multipath *m = ti->private;
1235         struct dm_mpath_io *mpio = map_context->ptr;
1236         struct pgpath *pgpath = mpio->pgpath;
1237         struct path_selector *ps;
1238         int r;
1239
1240         r  = do_end_io(m, clone, error, mpio);
1241         if (pgpath) {
1242                 ps = &pgpath->pg->ps;
1243                 if (ps->type->end_io)
1244                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1245         }
1246         mempool_free(mpio, m->mpio_pool);
1247
1248         return r;
1249 }
1250
1251 /*
1252  * Suspend can't complete until all the I/O is processed so if
1253  * the last path fails we must error any remaining I/O.
1254  * Note that if the freeze_bdev fails while suspending, the
1255  * queue_if_no_path state is lost - userspace should reset it.
1256  */
1257 static void multipath_presuspend(struct dm_target *ti)
1258 {
1259         struct multipath *m = (struct multipath *) ti->private;
1260
1261         queue_if_no_path(m, 0, 1);
1262 }
1263
1264 /*
1265  * Restore the queue_if_no_path setting.
1266  */
1267 static void multipath_resume(struct dm_target *ti)
1268 {
1269         struct multipath *m = (struct multipath *) ti->private;
1270         unsigned long flags;
1271
1272         spin_lock_irqsave(&m->lock, flags);
1273         m->queue_if_no_path = m->saved_queue_if_no_path;
1274         spin_unlock_irqrestore(&m->lock, flags);
1275 }
1276
1277 /*
1278  * Info output has the following format:
1279  * num_multipath_feature_args [multipath_feature_args]*
1280  * num_handler_status_args [handler_status_args]*
1281  * num_groups init_group_number
1282  *            [A|D|E num_ps_status_args [ps_status_args]*
1283  *             num_paths num_selector_args
1284  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1285  *
1286  * Table output has the following format (identical to the constructor string):
1287  * num_feature_args [features_args]*
1288  * num_handler_args hw_handler [hw_handler_args]*
1289  * num_groups init_group_number
1290  *     [priority selector-name num_ps_args [ps_args]*
1291  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1292  */
1293 static int multipath_status(struct dm_target *ti, status_type_t type,
1294                             char *result, unsigned int maxlen)
1295 {
1296         int sz = 0;
1297         unsigned long flags;
1298         struct multipath *m = (struct multipath *) ti->private;
1299         struct priority_group *pg;
1300         struct pgpath *p;
1301         unsigned pg_num;
1302         char state;
1303
1304         spin_lock_irqsave(&m->lock, flags);
1305
1306         /* Features */
1307         if (type == STATUSTYPE_INFO)
1308                 DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count);
1309         else {
1310                 DMEMIT("%u ", m->queue_if_no_path +
1311                               (m->pg_init_retries > 0) * 2);
1312                 if (m->queue_if_no_path)
1313                         DMEMIT("queue_if_no_path ");
1314                 if (m->pg_init_retries)
1315                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1316         }
1317
1318         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1319                 DMEMIT("0 ");
1320         else
1321                 DMEMIT("1 %s ", m->hw_handler_name);
1322
1323         DMEMIT("%u ", m->nr_priority_groups);
1324
1325         if (m->next_pg)
1326                 pg_num = m->next_pg->pg_num;
1327         else if (m->current_pg)
1328                 pg_num = m->current_pg->pg_num;
1329         else
1330                         pg_num = 1;
1331
1332         DMEMIT("%u ", pg_num);
1333
1334         switch (type) {
1335         case STATUSTYPE_INFO:
1336                 list_for_each_entry(pg, &m->priority_groups, list) {
1337                         if (pg->bypassed)
1338                                 state = 'D';    /* Disabled */
1339                         else if (pg == m->current_pg)
1340                                 state = 'A';    /* Currently Active */
1341                         else
1342                                 state = 'E';    /* Enabled */
1343
1344                         DMEMIT("%c ", state);
1345
1346                         if (pg->ps.type->status)
1347                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1348                                                           result + sz,
1349                                                           maxlen - sz);
1350                         else
1351                                 DMEMIT("0 ");
1352
1353                         DMEMIT("%u %u ", pg->nr_pgpaths,
1354                                pg->ps.type->info_args);
1355
1356                         list_for_each_entry(p, &pg->pgpaths, list) {
1357                                 DMEMIT("%s %s %u ", p->path.dev->name,
1358                                        p->is_active ? "A" : "F",
1359                                        p->fail_count);
1360                                 if (pg->ps.type->status)
1361                                         sz += pg->ps.type->status(&pg->ps,
1362                                               &p->path, type, result + sz,
1363                                               maxlen - sz);
1364                         }
1365                 }
1366                 break;
1367
1368         case STATUSTYPE_TABLE:
1369                 list_for_each_entry(pg, &m->priority_groups, list) {
1370                         DMEMIT("%s ", pg->ps.type->name);
1371
1372                         if (pg->ps.type->status)
1373                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1374                                                           result + sz,
1375                                                           maxlen - sz);
1376                         else
1377                                 DMEMIT("0 ");
1378
1379                         DMEMIT("%u %u ", pg->nr_pgpaths,
1380                                pg->ps.type->table_args);
1381
1382                         list_for_each_entry(p, &pg->pgpaths, list) {
1383                                 DMEMIT("%s ", p->path.dev->name);
1384                                 if (pg->ps.type->status)
1385                                         sz += pg->ps.type->status(&pg->ps,
1386                                               &p->path, type, result + sz,
1387                                               maxlen - sz);
1388                         }
1389                 }
1390                 break;
1391         }
1392
1393         spin_unlock_irqrestore(&m->lock, flags);
1394
1395         return 0;
1396 }
1397
1398 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1399 {
1400         int r;
1401         struct dm_dev *dev;
1402         struct multipath *m = (struct multipath *) ti->private;
1403         action_fn action;
1404
1405         if (argc == 1) {
1406                 if (!strnicmp(argv[0], MESG_STR("queue_if_no_path")))
1407                         return queue_if_no_path(m, 1, 0);
1408                 else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path")))
1409                         return queue_if_no_path(m, 0, 0);
1410         }
1411
1412         if (argc != 2)
1413                 goto error;
1414
1415         if (!strnicmp(argv[0], MESG_STR("disable_group")))
1416                 return bypass_pg_num(m, argv[1], 1);
1417         else if (!strnicmp(argv[0], MESG_STR("enable_group")))
1418                 return bypass_pg_num(m, argv[1], 0);
1419         else if (!strnicmp(argv[0], MESG_STR("switch_group")))
1420                 return switch_pg_num(m, argv[1]);
1421         else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
1422                 action = reinstate_path;
1423         else if (!strnicmp(argv[0], MESG_STR("fail_path")))
1424                 action = fail_path;
1425         else
1426                 goto error;
1427
1428         r = dm_get_device(ti, argv[1], ti->begin, ti->len,
1429                           dm_table_get_mode(ti->table), &dev);
1430         if (r) {
1431                 DMWARN("message: error getting device %s",
1432                        argv[1]);
1433                 return -EINVAL;
1434         }
1435
1436         r = action_dev(m, dev, action);
1437
1438         dm_put_device(ti, dev);
1439
1440         return r;
1441
1442 error:
1443         DMWARN("Unrecognised multipath message received.");
1444         return -EINVAL;
1445 }
1446
1447 static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
1448                            unsigned long arg)
1449 {
1450         struct multipath *m = (struct multipath *) ti->private;
1451         struct block_device *bdev = NULL;
1452         fmode_t mode = 0;
1453         unsigned long flags;
1454         int r = 0;
1455
1456         spin_lock_irqsave(&m->lock, flags);
1457
1458         if (!m->current_pgpath)
1459                 __choose_pgpath(m, 0);
1460
1461         if (m->current_pgpath) {
1462                 bdev = m->current_pgpath->path.dev->bdev;
1463                 mode = m->current_pgpath->path.dev->mode;
1464         }
1465
1466         if (m->queue_io)
1467                 r = -EAGAIN;
1468         else if (!bdev)
1469                 r = -EIO;
1470
1471         spin_unlock_irqrestore(&m->lock, flags);
1472
1473         return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
1474 }
1475
1476 static int multipath_iterate_devices(struct dm_target *ti,
1477                                      iterate_devices_callout_fn fn, void *data)
1478 {
1479         struct multipath *m = ti->private;
1480         struct priority_group *pg;
1481         struct pgpath *p;
1482         int ret = 0;
1483
1484         list_for_each_entry(pg, &m->priority_groups, list) {
1485                 list_for_each_entry(p, &pg->pgpaths, list) {
1486                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1487                         if (ret)
1488                                 goto out;
1489                 }
1490         }
1491
1492 out:
1493         return ret;
1494 }
1495
1496 static int __pgpath_busy(struct pgpath *pgpath)
1497 {
1498         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1499
1500         return dm_underlying_device_busy(q);
1501 }
1502
1503 /*
1504  * We return "busy", only when we can map I/Os but underlying devices
1505  * are busy (so even if we map I/Os now, the I/Os will wait on
1506  * the underlying queue).
1507  * In other words, if we want to kill I/Os or queue them inside us
1508  * due to map unavailability, we don't return "busy".  Otherwise,
1509  * dm core won't give us the I/Os and we can't do what we want.
1510  */
1511 static int multipath_busy(struct dm_target *ti)
1512 {
1513         int busy = 0, has_active = 0;
1514         struct multipath *m = ti->private;
1515         struct priority_group *pg;
1516         struct pgpath *pgpath;
1517         unsigned long flags;
1518
1519         spin_lock_irqsave(&m->lock, flags);
1520
1521         /* Guess which priority_group will be used at next mapping time */
1522         if (unlikely(!m->current_pgpath && m->next_pg))
1523                 pg = m->next_pg;
1524         else if (likely(m->current_pg))
1525                 pg = m->current_pg;
1526         else
1527                 /*
1528                  * We don't know which pg will be used at next mapping time.
1529                  * We don't call __choose_pgpath() here to avoid to trigger
1530                  * pg_init just by busy checking.
1531                  * So we don't know whether underlying devices we will be using
1532                  * at next mapping time are busy or not. Just try mapping.
1533                  */
1534                 goto out;
1535
1536         /*
1537          * If there is one non-busy active path at least, the path selector
1538          * will be able to select it. So we consider such a pg as not busy.
1539          */
1540         busy = 1;
1541         list_for_each_entry(pgpath, &pg->pgpaths, list)
1542                 if (pgpath->is_active) {
1543                         has_active = 1;
1544
1545                         if (!__pgpath_busy(pgpath)) {
1546                                 busy = 0;
1547                                 break;
1548                         }
1549                 }
1550
1551         if (!has_active)
1552                 /*
1553                  * No active path in this pg, so this pg won't be used and
1554                  * the current_pg will be changed at next mapping time.
1555                  * We need to try mapping to determine it.
1556                  */
1557                 busy = 0;
1558
1559 out:
1560         spin_unlock_irqrestore(&m->lock, flags);
1561
1562         return busy;
1563 }
1564
1565 /*-----------------------------------------------------------------
1566  * Module setup
1567  *---------------------------------------------------------------*/
1568 static struct target_type multipath_target = {
1569         .name = "multipath",
1570         .version = {1, 1, 0},
1571         .module = THIS_MODULE,
1572         .ctr = multipath_ctr,
1573         .dtr = multipath_dtr,
1574         .map_rq = multipath_map,
1575         .rq_end_io = multipath_end_io,
1576         .presuspend = multipath_presuspend,
1577         .resume = multipath_resume,
1578         .status = multipath_status,
1579         .message = multipath_message,
1580         .ioctl  = multipath_ioctl,
1581         .iterate_devices = multipath_iterate_devices,
1582         .busy = multipath_busy,
1583 };
1584
1585 static int __init dm_multipath_init(void)
1586 {
1587         int r;
1588
1589         /* allocate a slab for the dm_ios */
1590         _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
1591         if (!_mpio_cache)
1592                 return -ENOMEM;
1593
1594         r = dm_register_target(&multipath_target);
1595         if (r < 0) {
1596                 DMERR("register failed %d", r);
1597                 kmem_cache_destroy(_mpio_cache);
1598                 return -EINVAL;
1599         }
1600
1601         kmultipathd = create_workqueue("kmpathd");
1602         if (!kmultipathd) {
1603                 DMERR("failed to create workqueue kmpathd");
1604                 dm_unregister_target(&multipath_target);
1605                 kmem_cache_destroy(_mpio_cache);
1606                 return -ENOMEM;
1607         }
1608
1609         /*
1610          * A separate workqueue is used to handle the device handlers
1611          * to avoid overloading existing workqueue. Overloading the
1612          * old workqueue would also create a bottleneck in the
1613          * path of the storage hardware device activation.
1614          */
1615         kmpath_handlerd = create_singlethread_workqueue("kmpath_handlerd");
1616         if (!kmpath_handlerd) {
1617                 DMERR("failed to create workqueue kmpath_handlerd");
1618                 destroy_workqueue(kmultipathd);
1619                 dm_unregister_target(&multipath_target);
1620                 kmem_cache_destroy(_mpio_cache);
1621                 return -ENOMEM;
1622         }
1623
1624         DMINFO("version %u.%u.%u loaded",
1625                multipath_target.version[0], multipath_target.version[1],
1626                multipath_target.version[2]);
1627
1628         return r;
1629 }
1630
1631 static void __exit dm_multipath_exit(void)
1632 {
1633         destroy_workqueue(kmpath_handlerd);
1634         destroy_workqueue(kmultipathd);
1635
1636         dm_unregister_target(&multipath_target);
1637         kmem_cache_destroy(_mpio_cache);
1638 }
1639
1640 module_init(dm_multipath_init);
1641 module_exit(dm_multipath_exit);
1642
1643 MODULE_DESCRIPTION(DM_NAME " multipath target");
1644 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1645 MODULE_LICENSE("GPL");