drivers/nvme/host/multipath.c

   1 /*
   2  * Copyright (c) 2017-2018 Christoph Hellwig.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  */
  13
  14 #include <linux/moduleparam.h>
  15 #include <trace/events/block.h>
  16 #include "nvme.h"
  17
  18 static bool multipath = true;
  19 module_param(multipath, bool, 0444);
  20 MODULE_PARM_DESC(multipath,
  21         "turn on native support for multiple controllers per subsystem");
  22
  23 inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
  24 {
  25         return multipath && ctrl->subsys && (ctrl->subsys->cmic & (1 << 3));
  26 }
  27
  28 /*
  29  * If multipathing is enabled we need to always use the subsystem instance
  30  * number for numbering our devices to avoid conflicts between subsystems that
  31  * have multiple controllers and thus use the multipath-aware subsystem node
  32  * and those that have a single controller and use the controller node
  33  * directly.
  34  */
  35 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
  36                         struct nvme_ctrl *ctrl, int *flags)
  37 {
  38         if (!multipath) {
  39                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
  40         } else if (ns->head->disk) {
  41                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
  42                                 ctrl->cntlid, ns->head->instance);
  43                 *flags = GENHD_FL_HIDDEN;
  44         } else {
  45                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
  46                                 ns->head->instance);
  47         }
  48 }
  49
  50 void nvme_failover_req(struct request *req)
  51 {
  52         struct nvme_ns *ns = req->q->queuedata;
  53         u16 status = nvme_req(req)->status;
  54         unsigned long flags;
  55
  56         spin_lock_irqsave(&ns->head->requeue_lock, flags);
  57         blk_steal_bios(&ns->head->requeue_list, req);
  58         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
  59         blk_mq_end_request(req, 0);
  60
  61         switch (status & 0x7ff) {
  62         case NVME_SC_ANA_TRANSITION:
  63         case NVME_SC_ANA_INACCESSIBLE:
  64         case NVME_SC_ANA_PERSISTENT_LOSS:
  65                 /*
  66                  * If we got back an ANA error we know the controller is alive,
  67                  * but not ready to serve this namespaces.  The spec suggests
  68                  * we should update our general state here, but due to the fact
  69                  * that the admin and I/O queues are not serialized that is
  70                  * fundamentally racy.  So instead just clear the current path,
  71                  * mark the the path as pending and kick of a re-read of the ANA
  72                  * log page ASAP.
  73                  */
  74                 nvme_mpath_clear_current_path(ns);
  75                 if (ns->ctrl->ana_log_buf) {
  76                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  77                         queue_work(nvme_wq, &ns->ctrl->ana_work);
  78                 }
  79                 break;
  80         case NVME_SC_HOST_PATH_ERROR:
  81                 /*
  82                  * Temporary transport disruption in talking to the controller.
  83                  * Try to send on a new path.
  84                  */
  85                 nvme_mpath_clear_current_path(ns);
  86                 break;
  87         default:
  88                 /*
  89                  * Reset the controller for any non-ANA error as we don't know
  90                  * what caused the error.
  91                  */
  92                 nvme_reset_ctrl(ns->ctrl);
  93                 break;
  94         }
  95
  96         kblockd_schedule_work(&ns->head->requeue_work);
  97 }
  98
  99 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 100 {
 101         struct nvme_ns *ns;
 102
 103         down_read(&ctrl->namespaces_rwsem);
 104         list_for_each_entry(ns, &ctrl->namespaces, list) {
 105                 if (ns->head->disk)
 106                         kblockd_schedule_work(&ns->head->requeue_work);
 107         }
 108         up_read(&ctrl->namespaces_rwsem);
 109 }
 110
 111 static const char *nvme_ana_state_names[] = {
 112         [0]                             = "invalid state",
 113         [NVME_ANA_OPTIMIZED]            = "optimized",
 114         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 115         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 116         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 117         [NVME_ANA_CHANGE]               = "change",
 118 };
 119
 120 void nvme_mpath_clear_current_path(struct nvme_ns *ns)
 121 {
 122         struct nvme_ns_head *head = ns->head;
 123         int node;
 124
 125         if (!head)
 126                 return;
 127
 128         for_each_node(node) {
 129                 if (ns == rcu_access_pointer(head->current_path[node]))
 130                         rcu_assign_pointer(head->current_path[node], NULL);
 131         }
 132 }
 133
 134 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 135 {
 136         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 137         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 138
 139         list_for_each_entry_rcu(ns, &head->list, siblings) {
 140                 if (ns->ctrl->state != NVME_CTRL_LIVE ||
 141                     test_bit(NVME_NS_ANA_PENDING, &ns->flags))
 142                         continue;
 143
 144                 distance = node_distance(node, ns->ctrl->numa_node);
 145
 146                 switch (ns->ana_state) {
 147                 case NVME_ANA_OPTIMIZED:
 148                         if (distance < found_distance) {
 149                                 found_distance = distance;
 150                                 found = ns;
 151                         }
 152                         break;
 153                 case NVME_ANA_NONOPTIMIZED:
 154                         if (distance < fallback_distance) {
 155                                 fallback_distance = distance;
 156                                 fallback = ns;
 157                         }
 158                         break;
 159                 default:
 160                         break;
 161                 }
 162         }
 163
 164         if (!found)
 165                 found = fallback;
 166         if (found)
 167                 rcu_assign_pointer(head->current_path[node], found);
 168         return found;
 169 }
 170
 171 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 172 {
 173         return ns->ctrl->state == NVME_CTRL_LIVE &&
 174                 ns->ana_state == NVME_ANA_OPTIMIZED;
 175 }
 176
 177 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 178 {
 179         int node = numa_node_id();
 180         struct nvme_ns *ns;
 181
 182         ns = srcu_dereference(head->current_path[node], &head->srcu);
 183         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
 184                 ns = __nvme_find_path(head, node);
 185         return ns;
 186 }
 187
 188 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
 189                 struct bio *bio)
 190 {
 191         struct nvme_ns_head *head = q->queuedata;
 192         struct device *dev = disk_to_dev(head->disk);
 193         struct nvme_ns *ns;
 194         blk_qc_t ret = BLK_QC_T_NONE;
 195         int srcu_idx;
 196
 197         srcu_idx = srcu_read_lock(&head->srcu);
 198         ns = nvme_find_path(head);
 199         if (likely(ns)) {
 200                 bio->bi_disk = ns->disk;
 201                 bio->bi_opf |= REQ_NVME_MPATH;
 202                 trace_block_bio_remap(bio->bi_disk->queue, bio,
 203                                       disk_devt(ns->head->disk),
 204                                       bio->bi_iter.bi_sector);
 205                 ret = direct_make_request(bio);
 206         } else if (!list_empty_careful(&head->list)) {
 207                 dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");
 208
 209                 spin_lock_irq(&head->requeue_lock);
 210                 bio_list_add(&head->requeue_list, bio);
 211                 spin_unlock_irq(&head->requeue_lock);
 212         } else {
 213                 dev_warn_ratelimited(dev, "no path - failing I/O\n");
 214
 215                 bio->bi_status = BLK_STS_IOERR;
 216                 bio_endio(bio);
 217         }
 218
 219         srcu_read_unlock(&head->srcu, srcu_idx);
 220         return ret;
 221 }
 222
 223 static void nvme_requeue_work(struct work_struct *work)
 224 {
 225         struct nvme_ns_head *head =
 226                 container_of(work, struct nvme_ns_head, requeue_work);
 227         struct bio *bio, *next;
 228
 229         spin_lock_irq(&head->requeue_lock);
 230         next = bio_list_get(&head->requeue_list);
 231         spin_unlock_irq(&head->requeue_lock);
 232
 233         while ((bio = next) != NULL) {
 234                 next = bio->bi_next;
 235                 bio->bi_next = NULL;
 236
 237                 /*
 238                  * Reset disk to the mpath node and resubmit to select a new
 239                  * path.
 240                  */
 241                 bio->bi_disk = head->disk;
 242                 generic_make_request(bio);
 243         }
 244 }
 245
 246 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 247 {
 248         struct request_queue *q;
 249         bool vwc = false;
 250
 251         mutex_init(&head->lock);
 252         bio_list_init(&head->requeue_list);
 253         spin_lock_init(&head->requeue_lock);
 254         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 255
 256         /*
 257          * Add a multipath node if the subsystems supports multiple controllers.
 258          * We also do this for private namespaces as the namespace sharing data could
 259          * change after a rescan.
 260          */
 261         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
 262                 return 0;
 263
 264         q = blk_alloc_queue_node(GFP_KERNEL, ctrl->numa_node);
 265         if (!q)
 266                 goto out;
 267         q->queuedata = head;
 268         blk_queue_make_request(q, nvme_ns_head_make_request);
 269         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 270         /* set to a default value for 512 until disk is validated */
 271         blk_queue_logical_block_size(q, 512);
 272         blk_set_stacking_limits(&q->limits);
 273
 274         /* we need to propagate up the VMC settings */
 275         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 276                 vwc = true;
 277         blk_queue_write_cache(q, vwc, vwc);
 278
 279         head->disk = alloc_disk(0);
 280         if (!head->disk)
 281                 goto out_cleanup_queue;
 282         head->disk->fops = &nvme_ns_head_ops;
 283         head->disk->private_data = head;
 284         head->disk->queue = q;
 285         head->disk->flags = GENHD_FL_EXT_DEVT;
 286         sprintf(head->disk->disk_name, "nvme%dn%d",
 287                         ctrl->subsys->instance, head->instance);
 288         return 0;
 289
 290 out_cleanup_queue:
 291         blk_cleanup_queue(q);
 292 out:
 293         return -ENOMEM;
 294 }
 295
 296 static void nvme_mpath_set_live(struct nvme_ns *ns)
 297 {
 298         struct nvme_ns_head *head = ns->head;
 299
 300         lockdep_assert_held(&ns->head->lock);
 301
 302         if (!head->disk)
 303                 return;
 304
 305         if (!(head->disk->flags & GENHD_FL_UP))
 306                 device_add_disk(&head->subsys->dev, head->disk,
 307                                 nvme_ns_id_attr_groups);
 308
 309         if (nvme_path_is_optimized(ns)) {
 310                 int node, srcu_idx;
 311
 312                 srcu_idx = srcu_read_lock(&head->srcu);
 313                 for_each_node(node)
 314                         __nvme_find_path(head, node);
 315                 srcu_read_unlock(&head->srcu, srcu_idx);
 316         }
 317
 318         kblockd_schedule_work(&ns->head->requeue_work);
 319 }
 320
 321 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 322                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 323                         void *))
 324 {
 325         void *base = ctrl->ana_log_buf;
 326         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 327         int error, i;
 328
 329         lockdep_assert_held(&ctrl->ana_lock);
 330
 331         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 332                 struct nvme_ana_group_desc *desc = base + offset;
 333                 u32 nr_nsids = le32_to_cpu(desc->nnsids);
 334                 size_t nsid_buf_size = nr_nsids * sizeof(__le32);
 335
 336                 if (WARN_ON_ONCE(desc->grpid == 0))
 337                         return -EINVAL;
 338                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 339                         return -EINVAL;
 340                 if (WARN_ON_ONCE(desc->state == 0))
 341                         return -EINVAL;
 342                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 343                         return -EINVAL;
 344
 345                 offset += sizeof(*desc);
 346                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 347                         return -EINVAL;
 348
 349                 error = cb(ctrl, desc, data);
 350                 if (error)
 351                         return error;
 352
 353                 offset += nsid_buf_size;
 354                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 355                         return -EINVAL;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 362 {
 363         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 364 }
 365
 366 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 367                 struct nvme_ns *ns)
 368 {
 369         enum nvme_ana_state old;
 370
 371         mutex_lock(&ns->head->lock);
 372         old = ns->ana_state;
 373         ns->ana_grpid = le32_to_cpu(desc->grpid);
 374         ns->ana_state = desc->state;
 375         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 376
 377         if (nvme_state_is_live(ns->ana_state) && !nvme_state_is_live(old))
 378                 nvme_mpath_set_live(ns);
 379         mutex_unlock(&ns->head->lock);
 380 }
 381
 382 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 383                 struct nvme_ana_group_desc *desc, void *data)
 384 {
 385         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 386         unsigned *nr_change_groups = data;
 387         struct nvme_ns *ns;
 388
 389         dev_info(ctrl->device, "ANA group %d: %s.\n",
 390                         le32_to_cpu(desc->grpid),
 391                         nvme_ana_state_names[desc->state]);
 392
 393         if (desc->state == NVME_ANA_CHANGE)
 394                 (*nr_change_groups)++;
 395
 396         if (!nr_nsids)
 397                 return 0;
 398
 399         down_write(&ctrl->namespaces_rwsem);
 400         list_for_each_entry(ns, &ctrl->namespaces, list) {
 401                 if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
 402                         continue;
 403                 nvme_update_ns_ana_state(desc, ns);
 404                 if (++n == nr_nsids)
 405                         break;
 406         }
 407         up_write(&ctrl->namespaces_rwsem);
 408         WARN_ON_ONCE(n < nr_nsids);
 409         return 0;
 410 }
 411
 412 static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 413 {
 414         u32 nr_change_groups = 0;
 415         int error;
 416
 417         mutex_lock(&ctrl->ana_lock);
 418         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
 419                         groups_only ? NVME_ANA_LOG_RGO : 0,
 420                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 421         if (error) {
 422                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 423                 goto out_unlock;
 424         }
 425
 426         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 427                         nvme_update_ana_state);
 428         if (error)
 429                 goto out_unlock;
 430
 431         /*
 432          * In theory we should have an ANATT timer per group as they might enter
 433          * the change state at different times.  But that is a lot of overhead
 434          * just to protect against a target that keeps entering new changes
 435          * states while never finishing previous ones.  But we'll still
 436          * eventually time out once all groups are in change state, so this
 437          * isn't a big deal.
 438          *
 439          * We also double the ANATT value to provide some slack for transports
 440          * or AEN processing overhead.
 441          */
 442         if (nr_change_groups)
 443                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 444         else
 445                 del_timer_sync(&ctrl->anatt_timer);
 446 out_unlock:
 447         mutex_unlock(&ctrl->ana_lock);
 448         return error;
 449 }
 450
 451 static void nvme_ana_work(struct work_struct *work)
 452 {
 453         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 454
 455         nvme_read_ana_log(ctrl, false);
 456 }
 457
 458 static void nvme_anatt_timeout(struct timer_list *t)
 459 {
 460         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 461
 462         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 463         nvme_reset_ctrl(ctrl);
 464 }
 465
 466 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 467 {
 468         if (!nvme_ctrl_use_ana(ctrl))
 469                 return;
 470         del_timer_sync(&ctrl->anatt_timer);
 471         cancel_work_sync(&ctrl->ana_work);
 472 }
 473
 474 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 475                 char *buf)
 476 {
 477         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 478 }
 479 DEVICE_ATTR_RO(ana_grpid);
 480
 481 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 482                 char *buf)
 483 {
 484         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 485
 486         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 487 }
 488 DEVICE_ATTR_RO(ana_state);
 489
 490 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
 491                 struct nvme_ana_group_desc *desc, void *data)
 492 {
 493         struct nvme_ns *ns = data;
 494
 495         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
 496                 nvme_update_ns_ana_state(desc, ns);
 497                 return -ENXIO; /* just break out of the loop */
 498         }
 499
 500         return 0;
 501 }
 502
 503 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 504 {
 505         if (nvme_ctrl_use_ana(ns->ctrl)) {
 506                 mutex_lock(&ns->ctrl->ana_lock);
 507                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 508                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
 509                 mutex_unlock(&ns->ctrl->ana_lock);
 510         } else {
 511                 mutex_lock(&ns->head->lock);
 512                 ns->ana_state = NVME_ANA_OPTIMIZED;
 513                 nvme_mpath_set_live(ns);
 514                 mutex_unlock(&ns->head->lock);
 515         }
 516 }
 517
 518 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 519 {
 520         if (!head->disk)
 521                 return;
 522         if (head->disk->flags & GENHD_FL_UP)
 523                 del_gendisk(head->disk);
 524         blk_set_queue_dying(head->disk->queue);
 525         /* make sure all pending bios are cleaned up */
 526         kblockd_schedule_work(&head->requeue_work);
 527         flush_work(&head->requeue_work);
 528         blk_cleanup_queue(head->disk->queue);
 529         put_disk(head->disk);
 530 }
 531
 532 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 533 {
 534         int error;
 535
 536         if (!nvme_ctrl_use_ana(ctrl))
 537                 return 0;
 538
 539         ctrl->anacap = id->anacap;
 540         ctrl->anatt = id->anatt;
 541         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 542         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 543
 544         mutex_init(&ctrl->ana_lock);
 545         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 546         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 547                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
 548         ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
 549
 550         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
 551                 dev_err(ctrl->device,
 552                         "ANA log page size (%zd) larger than MDTS (%d).\n",
 553                         ctrl->ana_log_size,
 554                         ctrl->max_hw_sectors << SECTOR_SHIFT);
 555                 dev_err(ctrl->device, "disabling ANA support.\n");
 556                 return 0;
 557         }
 558
 559         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 560         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
 561         if (!ctrl->ana_log_buf) {
 562                 error = -ENOMEM;
 563                 goto out;
 564         }
 565
 566         error = nvme_read_ana_log(ctrl, true);
 567         if (error)
 568                 goto out_free_ana_log_buf;
 569         return 0;
 570 out_free_ana_log_buf:
 571         kfree(ctrl->ana_log_buf);
 572         ctrl->ana_log_buf = NULL;
 573 out:
 574         return error;
 575 }
 576
 577 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 578 {
 579         kfree(ctrl->ana_log_buf);
 580         ctrl->ana_log_buf = NULL;
 581 }
 582