drivers/nvme/target/core.c

   1 /*
   2  * Common code for the NVMe target.
   3  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15 #include <linux/module.h>
  16 #include <linux/random.h>
  17 #include <linux/rculist.h>
  18
  19 #include "nvmet.h"
  20
  21 static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
  22 static DEFINE_IDA(cntlid_ida);
  23
  24 /*
  25  * This read/write semaphore is used to synchronize access to configuration
  26  * information on a target system that will result in discovery log page
  27  * information change for at least one host.
  28  * The full list of resources to protected by this semaphore is:
  29  *
  30  *  - subsystems list
  31  *  - per-subsystem allowed hosts list
  32  *  - allow_any_host subsystem attribute
  33  *  - nvmet_genctr
  34  *  - the nvmet_transports array
  35  *
  36  * When updating any of those lists/structures write lock should be obtained,
  37  * while when reading (popolating discovery log page or checking host-subsystem
  38  * link) read lock is obtained to allow concurrent reads.
  39  */
  40 DECLARE_RWSEM(nvmet_config_sem);
  41
  42 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
  43                 const char *subsysnqn);
  44
  45 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf,
  46                 size_t len)
  47 {
  48         if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len)
  49                 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
  50         return 0;
  51 }
  52
  53 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len)
  54 {
  55         if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len)
  56                 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
  57         return 0;
  58 }
  59
  60 u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len)
  61 {
  62         if (sg_zero_buffer(req->sg, req->sg_cnt, len, off) != len)
  63                 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
  64         return 0;
  65 }
  66
  67 static unsigned int nvmet_max_nsid(struct nvmet_subsys *subsys)
  68 {
  69         struct nvmet_ns *ns;
  70
  71         if (list_empty(&subsys->namespaces))
  72                 return 0;
  73
  74         ns = list_last_entry(&subsys->namespaces, struct nvmet_ns, dev_link);
  75         return ns->nsid;
  76 }
  77
  78 static u32 nvmet_async_event_result(struct nvmet_async_event *aen)
  79 {
  80         return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16);
  81 }
  82
  83 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)
  84 {
  85         struct nvmet_req *req;
  86
  87         while (1) {
  88                 mutex_lock(&ctrl->lock);
  89                 if (!ctrl->nr_async_event_cmds) {
  90                         mutex_unlock(&ctrl->lock);
  91                         return;
  92                 }
  93
  94                 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
  95                 mutex_unlock(&ctrl->lock);
  96                 nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR);
  97         }
  98 }
  99
 100 static void nvmet_async_event_work(struct work_struct *work)
 101 {
 102         struct nvmet_ctrl *ctrl =
 103                 container_of(work, struct nvmet_ctrl, async_event_work);
 104         struct nvmet_async_event *aen;
 105         struct nvmet_req *req;
 106
 107         while (1) {
 108                 mutex_lock(&ctrl->lock);
 109                 aen = list_first_entry_or_null(&ctrl->async_events,
 110                                 struct nvmet_async_event, entry);
 111                 if (!aen || !ctrl->nr_async_event_cmds) {
 112                         mutex_unlock(&ctrl->lock);
 113                         return;
 114                 }
 115
 116                 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
 117                 nvmet_set_result(req, nvmet_async_event_result(aen));
 118
 119                 list_del(&aen->entry);
 120                 kfree(aen);
 121
 122                 mutex_unlock(&ctrl->lock);
 123                 nvmet_req_complete(req, 0);
 124         }
 125 }
 126
 127 static void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
 128                 u8 event_info, u8 log_page)
 129 {
 130         struct nvmet_async_event *aen;
 131
 132         aen = kmalloc(sizeof(*aen), GFP_KERNEL);
 133         if (!aen)
 134                 return;
 135
 136         aen->event_type = event_type;
 137         aen->event_info = event_info;
 138         aen->log_page = log_page;
 139
 140         mutex_lock(&ctrl->lock);
 141         list_add_tail(&aen->entry, &ctrl->async_events);
 142         mutex_unlock(&ctrl->lock);
 143
 144         schedule_work(&ctrl->async_event_work);
 145 }
 146
 147 static bool nvmet_aen_disabled(struct nvmet_ctrl *ctrl, u32 aen)
 148 {
 149         if (!(READ_ONCE(ctrl->aen_enabled) & aen))
 150                 return true;
 151         return test_and_set_bit(aen, &ctrl->aen_masked);
 152 }
 153
 154 static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid)
 155 {
 156         u32 i;
 157
 158         mutex_lock(&ctrl->lock);
 159         if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES)
 160                 goto out_unlock;
 161
 162         for (i = 0; i < ctrl->nr_changed_ns; i++) {
 163                 if (ctrl->changed_ns_list[i] == nsid)
 164                         goto out_unlock;
 165         }
 166
 167         if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) {
 168                 ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff);
 169                 ctrl->nr_changed_ns = U32_MAX;
 170                 goto out_unlock;
 171         }
 172
 173         ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid;
 174 out_unlock:
 175         mutex_unlock(&ctrl->lock);
 176 }
 177
 178 static void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid)
 179 {
 180         struct nvmet_ctrl *ctrl;
 181
 182         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 183                 nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid));
 184                 if (nvmet_aen_disabled(ctrl, NVME_AEN_CFG_NS_ATTR))
 185                         continue;
 186                 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
 187                                 NVME_AER_NOTICE_NS_CHANGED,
 188                                 NVME_LOG_CHANGED_NS);
 189         }
 190 }
 191
 192 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops)
 193 {
 194         int ret = 0;
 195
 196         down_write(&nvmet_config_sem);
 197         if (nvmet_transports[ops->type])
 198                 ret = -EINVAL;
 199         else
 200                 nvmet_transports[ops->type] = ops;
 201         up_write(&nvmet_config_sem);
 202
 203         return ret;
 204 }
 205 EXPORT_SYMBOL_GPL(nvmet_register_transport);
 206
 207 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops)
 208 {
 209         down_write(&nvmet_config_sem);
 210         nvmet_transports[ops->type] = NULL;
 211         up_write(&nvmet_config_sem);
 212 }
 213 EXPORT_SYMBOL_GPL(nvmet_unregister_transport);
 214
 215 int nvmet_enable_port(struct nvmet_port *port)
 216 {
 217         const struct nvmet_fabrics_ops *ops;
 218         int ret;
 219
 220         lockdep_assert_held(&nvmet_config_sem);
 221
 222         ops = nvmet_transports[port->disc_addr.trtype];
 223         if (!ops) {
 224                 up_write(&nvmet_config_sem);
 225                 request_module("nvmet-transport-%d", port->disc_addr.trtype);
 226                 down_write(&nvmet_config_sem);
 227                 ops = nvmet_transports[port->disc_addr.trtype];
 228                 if (!ops) {
 229                         pr_err("transport type %d not supported\n",
 230                                 port->disc_addr.trtype);
 231                         return -EINVAL;
 232                 }
 233         }
 234
 235         if (!try_module_get(ops->owner))
 236                 return -EINVAL;
 237
 238         ret = ops->add_port(port);
 239         if (ret) {
 240                 module_put(ops->owner);
 241                 return ret;
 242         }
 243
 244         port->enabled = true;
 245         return 0;
 246 }
 247
 248 void nvmet_disable_port(struct nvmet_port *port)
 249 {
 250         const struct nvmet_fabrics_ops *ops;
 251
 252         lockdep_assert_held(&nvmet_config_sem);
 253
 254         port->enabled = false;
 255
 256         ops = nvmet_transports[port->disc_addr.trtype];
 257         ops->remove_port(port);
 258         module_put(ops->owner);
 259 }
 260
 261 static void nvmet_keep_alive_timer(struct work_struct *work)
 262 {
 263         struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),
 264                         struct nvmet_ctrl, ka_work);
 265
 266         pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n",
 267                 ctrl->cntlid, ctrl->kato);
 268
 269         nvmet_ctrl_fatal_error(ctrl);
 270 }
 271
 272 static void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)
 273 {
 274         pr_debug("ctrl %d start keep-alive timer for %d secs\n",
 275                 ctrl->cntlid, ctrl->kato);
 276
 277         INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);
 278         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
 279 }
 280
 281 static void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)
 282 {
 283         pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid);
 284
 285         cancel_delayed_work_sync(&ctrl->ka_work);
 286 }
 287
 288 static struct nvmet_ns *__nvmet_find_namespace(struct nvmet_ctrl *ctrl,
 289                 __le32 nsid)
 290 {
 291         struct nvmet_ns *ns;
 292
 293         list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {
 294                 if (ns->nsid == le32_to_cpu(nsid))
 295                         return ns;
 296         }
 297
 298         return NULL;
 299 }
 300
 301 struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid)
 302 {
 303         struct nvmet_ns *ns;
 304
 305         rcu_read_lock();
 306         ns = __nvmet_find_namespace(ctrl, nsid);
 307         if (ns)
 308                 percpu_ref_get(&ns->ref);
 309         rcu_read_unlock();
 310
 311         return ns;
 312 }
 313
 314 static void nvmet_destroy_namespace(struct percpu_ref *ref)
 315 {
 316         struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref);
 317
 318         complete(&ns->disable_done);
 319 }
 320
 321 void nvmet_put_namespace(struct nvmet_ns *ns)
 322 {
 323         percpu_ref_put(&ns->ref);
 324 }
 325
 326 static void nvmet_ns_dev_disable(struct nvmet_ns *ns)
 327 {
 328         nvmet_bdev_ns_disable(ns);
 329         nvmet_file_ns_disable(ns);
 330 }
 331
 332 int nvmet_ns_enable(struct nvmet_ns *ns)
 333 {
 334         struct nvmet_subsys *subsys = ns->subsys;
 335         int ret = 0;
 336
 337         mutex_lock(&subsys->lock);
 338         if (ns->enabled)
 339                 goto out_unlock;
 340
 341         ret = nvmet_bdev_ns_enable(ns);
 342         if (ret)
 343                 ret = nvmet_file_ns_enable(ns);
 344         if (ret)
 345                 goto out_unlock;
 346
 347         ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace,
 348                                 0, GFP_KERNEL);
 349         if (ret)
 350                 goto out_dev_put;
 351
 352         if (ns->nsid > subsys->max_nsid)
 353                 subsys->max_nsid = ns->nsid;
 354
 355         /*
 356          * The namespaces list needs to be sorted to simplify the implementation
 357          * of the Identify Namepace List subcommand.
 358          */
 359         if (list_empty(&subsys->namespaces)) {
 360                 list_add_tail_rcu(&ns->dev_link, &subsys->namespaces);
 361         } else {
 362                 struct nvmet_ns *old;
 363
 364                 list_for_each_entry_rcu(old, &subsys->namespaces, dev_link) {
 365                         BUG_ON(ns->nsid == old->nsid);
 366                         if (ns->nsid < old->nsid)
 367                                 break;
 368                 }
 369
 370                 list_add_tail_rcu(&ns->dev_link, &old->dev_link);
 371         }
 372
 373         nvmet_ns_changed(subsys, ns->nsid);
 374         ns->enabled = true;
 375         ret = 0;
 376 out_unlock:
 377         mutex_unlock(&subsys->lock);
 378         return ret;
 379 out_dev_put:
 380         nvmet_ns_dev_disable(ns);
 381         goto out_unlock;
 382 }
 383
 384 void nvmet_ns_disable(struct nvmet_ns *ns)
 385 {
 386         struct nvmet_subsys *subsys = ns->subsys;
 387
 388         mutex_lock(&subsys->lock);
 389         if (!ns->enabled)
 390                 goto out_unlock;
 391
 392         ns->enabled = false;
 393         list_del_rcu(&ns->dev_link);
 394         if (ns->nsid == subsys->max_nsid)
 395                 subsys->max_nsid = nvmet_max_nsid(subsys);
 396         mutex_unlock(&subsys->lock);
 397
 398         /*
 399          * Now that we removed the namespaces from the lookup list, we
 400          * can kill the per_cpu ref and wait for any remaining references
 401          * to be dropped, as well as a RCU grace period for anyone only
 402          * using the namepace under rcu_read_lock().  Note that we can't
 403          * use call_rcu here as we need to ensure the namespaces have
 404          * been fully destroyed before unloading the module.
 405          */
 406         percpu_ref_kill(&ns->ref);
 407         synchronize_rcu();
 408         wait_for_completion(&ns->disable_done);
 409         percpu_ref_exit(&ns->ref);
 410
 411         mutex_lock(&subsys->lock);
 412         nvmet_ns_changed(subsys, ns->nsid);
 413         nvmet_ns_dev_disable(ns);
 414 out_unlock:
 415         mutex_unlock(&subsys->lock);
 416 }
 417
 418 void nvmet_ns_free(struct nvmet_ns *ns)
 419 {
 420         nvmet_ns_disable(ns);
 421
 422         kfree(ns->device_path);
 423         kfree(ns);
 424 }
 425
 426 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid)
 427 {
 428         struct nvmet_ns *ns;
 429
 430         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
 431         if (!ns)
 432                 return NULL;
 433
 434         INIT_LIST_HEAD(&ns->dev_link);
 435         init_completion(&ns->disable_done);
 436
 437         ns->nsid = nsid;
 438         ns->subsys = subsys;
 439         uuid_gen(&ns->uuid);
 440
 441         return ns;
 442 }
 443
 444 static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 445 {
 446         u32 old_sqhd, new_sqhd;
 447         u16 sqhd;
 448
 449         if (status)
 450                 nvmet_set_status(req, status);
 451
 452         if (req->sq->size) {
 453                 do {
 454                         old_sqhd = req->sq->sqhd;
 455                         new_sqhd = (old_sqhd + 1) % req->sq->size;
 456                 } while (cmpxchg(&req->sq->sqhd, old_sqhd, new_sqhd) !=
 457                                         old_sqhd);
 458         }
 459         sqhd = req->sq->sqhd & 0x0000FFFF;
 460         req->rsp->sq_head = cpu_to_le16(sqhd);
 461         req->rsp->sq_id = cpu_to_le16(req->sq->qid);
 462         req->rsp->command_id = req->cmd->common.command_id;
 463
 464         if (req->ns)
 465                 nvmet_put_namespace(req->ns);
 466         req->ops->queue_response(req);
 467 }
 468
 469 void nvmet_req_complete(struct nvmet_req *req, u16 status)
 470 {
 471         __nvmet_req_complete(req, status);
 472         percpu_ref_put(&req->sq->ref);
 473 }
 474 EXPORT_SYMBOL_GPL(nvmet_req_complete);
 475
 476 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq,
 477                 u16 qid, u16 size)
 478 {
 479         cq->qid = qid;
 480         cq->size = size;
 481
 482         ctrl->cqs[qid] = cq;
 483 }
 484
 485 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,
 486                 u16 qid, u16 size)
 487 {
 488         sq->sqhd = 0;
 489         sq->qid = qid;
 490         sq->size = size;
 491
 492         ctrl->sqs[qid] = sq;
 493 }
 494
 495 static void nvmet_confirm_sq(struct percpu_ref *ref)
 496 {
 497         struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);
 498
 499         complete(&sq->confirm_done);
 500 }
 501
 502 void nvmet_sq_destroy(struct nvmet_sq *sq)
 503 {
 504         /*
 505          * If this is the admin queue, complete all AERs so that our
 506          * queue doesn't have outstanding requests on it.
 507          */
 508         if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq)
 509                 nvmet_async_events_free(sq->ctrl);
 510         percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq);
 511         wait_for_completion(&sq->confirm_done);
 512         wait_for_completion(&sq->free_done);
 513         percpu_ref_exit(&sq->ref);
 514
 515         if (sq->ctrl) {
 516                 nvmet_ctrl_put(sq->ctrl);
 517                 sq->ctrl = NULL; /* allows reusing the queue later */
 518         }
 519 }
 520 EXPORT_SYMBOL_GPL(nvmet_sq_destroy);
 521
 522 static void nvmet_sq_free(struct percpu_ref *ref)
 523 {
 524         struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);
 525
 526         complete(&sq->free_done);
 527 }
 528
 529 int nvmet_sq_init(struct nvmet_sq *sq)
 530 {
 531         int ret;
 532
 533         ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL);
 534         if (ret) {
 535                 pr_err("percpu_ref init failed!\n");
 536                 return ret;
 537         }
 538         init_completion(&sq->free_done);
 539         init_completion(&sq->confirm_done);
 540
 541         return 0;
 542 }
 543 EXPORT_SYMBOL_GPL(nvmet_sq_init);
 544
 545 static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 546 {
 547         struct nvme_command *cmd = req->cmd;
 548         u16 ret;
 549
 550         ret = nvmet_check_ctrl_status(req, cmd);
 551         if (unlikely(ret))
 552                 return ret;
 553
 554         req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);
 555         if (unlikely(!req->ns))
 556                 return NVME_SC_INVALID_NS | NVME_SC_DNR;
 557
 558         if (req->ns->file)
 559                 return nvmet_file_parse_io_cmd(req);
 560         else
 561                 return nvmet_bdev_parse_io_cmd(req);
 562 }
 563
 564 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
 565                 struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops)
 566 {
 567         u8 flags = req->cmd->common.flags;
 568         u16 status;
 569
 570         req->cq = cq;
 571         req->sq = sq;
 572         req->ops = ops;
 573         req->sg = NULL;
 574         req->sg_cnt = 0;
 575         req->transfer_len = 0;
 576         req->rsp->status = 0;
 577         req->ns = NULL;
 578
 579         /* no support for fused commands yet */
 580         if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) {
 581                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 582                 goto fail;
 583         }
 584
 585         /*
 586          * For fabrics, PSDT field shall describe metadata pointer (MPTR) that
 587          * contains an address of a single contiguous physical buffer that is
 588          * byte aligned.
 589          */
 590         if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF)) {
 591                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 592                 goto fail;
 593         }
 594
 595         if (unlikely(!req->sq->ctrl))
 596                 /* will return an error for any Non-connect command: */
 597                 status = nvmet_parse_connect_cmd(req);
 598         else if (likely(req->sq->qid != 0))
 599                 status = nvmet_parse_io_cmd(req);
 600         else if (req->cmd->common.opcode == nvme_fabrics_command)
 601                 status = nvmet_parse_fabrics_cmd(req);
 602         else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC)
 603                 status = nvmet_parse_discovery_cmd(req);
 604         else
 605                 status = nvmet_parse_admin_cmd(req);
 606
 607         if (status)
 608                 goto fail;
 609
 610         if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {
 611                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 612                 goto fail;
 613         }
 614
 615         return true;
 616
 617 fail:
 618         __nvmet_req_complete(req, status);
 619         return false;
 620 }
 621 EXPORT_SYMBOL_GPL(nvmet_req_init);
 622
 623 void nvmet_req_uninit(struct nvmet_req *req)
 624 {
 625         percpu_ref_put(&req->sq->ref);
 626         if (req->ns)
 627                 nvmet_put_namespace(req->ns);
 628 }
 629 EXPORT_SYMBOL_GPL(nvmet_req_uninit);
 630
 631 void nvmet_req_execute(struct nvmet_req *req)
 632 {
 633         if (unlikely(req->data_len != req->transfer_len))
 634                 nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR);
 635         else
 636                 req->execute(req);
 637 }
 638 EXPORT_SYMBOL_GPL(nvmet_req_execute);
 639
 640 static inline bool nvmet_cc_en(u32 cc)
 641 {
 642         return (cc >> NVME_CC_EN_SHIFT) & 0x1;
 643 }
 644
 645 static inline u8 nvmet_cc_css(u32 cc)
 646 {
 647         return (cc >> NVME_CC_CSS_SHIFT) & 0x7;
 648 }
 649
 650 static inline u8 nvmet_cc_mps(u32 cc)
 651 {
 652         return (cc >> NVME_CC_MPS_SHIFT) & 0xf;
 653 }
 654
 655 static inline u8 nvmet_cc_ams(u32 cc)
 656 {
 657         return (cc >> NVME_CC_AMS_SHIFT) & 0x7;
 658 }
 659
 660 static inline u8 nvmet_cc_shn(u32 cc)
 661 {
 662         return (cc >> NVME_CC_SHN_SHIFT) & 0x3;
 663 }
 664
 665 static inline u8 nvmet_cc_iosqes(u32 cc)
 666 {
 667         return (cc >> NVME_CC_IOSQES_SHIFT) & 0xf;
 668 }
 669
 670 static inline u8 nvmet_cc_iocqes(u32 cc)
 671 {
 672         return (cc >> NVME_CC_IOCQES_SHIFT) & 0xf;
 673 }
 674
 675 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)
 676 {
 677         lockdep_assert_held(&ctrl->lock);
 678
 679         if (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES ||
 680             nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES ||
 681             nvmet_cc_mps(ctrl->cc) != 0 ||
 682             nvmet_cc_ams(ctrl->cc) != 0 ||
 683             nvmet_cc_css(ctrl->cc) != 0) {
 684                 ctrl->csts = NVME_CSTS_CFS;
 685                 return;
 686         }
 687
 688         ctrl->csts = NVME_CSTS_RDY;
 689
 690         /*
 691          * Controllers that are not yet enabled should not really enforce the
 692          * keep alive timeout, but we still want to track a timeout and cleanup
 693          * in case a host died before it enabled the controller.  Hence, simply
 694          * reset the keep alive timer when the controller is enabled.
 695          */
 696         mod_delayed_work(system_wq, &ctrl->ka_work, ctrl->kato * HZ);
 697 }
 698
 699 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl)
 700 {
 701         lockdep_assert_held(&ctrl->lock);
 702
 703         /* XXX: tear down queues? */
 704         ctrl->csts &= ~NVME_CSTS_RDY;
 705         ctrl->cc = 0;
 706 }
 707
 708 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new)
 709 {
 710         u32 old;
 711
 712         mutex_lock(&ctrl->lock);
 713         old = ctrl->cc;
 714         ctrl->cc = new;
 715
 716         if (nvmet_cc_en(new) && !nvmet_cc_en(old))
 717                 nvmet_start_ctrl(ctrl);
 718         if (!nvmet_cc_en(new) && nvmet_cc_en(old))
 719                 nvmet_clear_ctrl(ctrl);
 720         if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) {
 721                 nvmet_clear_ctrl(ctrl);
 722                 ctrl->csts |= NVME_CSTS_SHST_CMPLT;
 723         }
 724         if (!nvmet_cc_shn(new) && nvmet_cc_shn(old))
 725                 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;
 726         mutex_unlock(&ctrl->lock);
 727 }
 728
 729 static void nvmet_init_cap(struct nvmet_ctrl *ctrl)
 730 {
 731         /* command sets supported: NVMe command set: */
 732         ctrl->cap = (1ULL << 37);
 733         /* CC.EN timeout in 500msec units: */
 734         ctrl->cap |= (15ULL << 24);
 735         /* maximum queue entries supported: */
 736         ctrl->cap |= NVMET_QUEUE_SIZE - 1;
 737 }
 738
 739 u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid,
 740                 struct nvmet_req *req, struct nvmet_ctrl **ret)
 741 {
 742         struct nvmet_subsys *subsys;
 743         struct nvmet_ctrl *ctrl;
 744         u16 status = 0;
 745
 746         subsys = nvmet_find_get_subsys(req->port, subsysnqn);
 747         if (!subsys) {
 748                 pr_warn("connect request for invalid subsystem %s!\n",
 749                         subsysnqn);
 750                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 751                 return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 752         }
 753
 754         mutex_lock(&subsys->lock);
 755         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 756                 if (ctrl->cntlid == cntlid) {
 757                         if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) {
 758                                 pr_warn("hostnqn mismatch.\n");
 759                                 continue;
 760                         }
 761                         if (!kref_get_unless_zero(&ctrl->ref))
 762                                 continue;
 763
 764                         *ret = ctrl;
 765                         goto out;
 766                 }
 767         }
 768
 769         pr_warn("could not find controller %d for subsys %s / host %s\n",
 770                 cntlid, subsysnqn, hostnqn);
 771         req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid);
 772         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 773
 774 out:
 775         mutex_unlock(&subsys->lock);
 776         nvmet_subsys_put(subsys);
 777         return status;
 778 }
 779
 780 u16 nvmet_check_ctrl_status(struct nvmet_req *req, struct nvme_command *cmd)
 781 {
 782         if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {
 783                 pr_err("got cmd %d while CC.EN == 0 on qid = %d\n",
 784                        cmd->common.opcode, req->sq->qid);
 785                 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
 786         }
 787
 788         if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {
 789                 pr_err("got cmd %d while CSTS.RDY == 0 on qid = %d\n",
 790                        cmd->common.opcode, req->sq->qid);
 791                 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
 792         }
 793         return 0;
 794 }
 795
 796 static bool __nvmet_host_allowed(struct nvmet_subsys *subsys,
 797                 const char *hostnqn)
 798 {
 799         struct nvmet_host_link *p;
 800
 801         if (subsys->allow_any_host)
 802                 return true;
 803
 804         list_for_each_entry(p, &subsys->hosts, entry) {
 805                 if (!strcmp(nvmet_host_name(p->host), hostnqn))
 806                         return true;
 807         }
 808
 809         return false;
 810 }
 811
 812 static bool nvmet_host_discovery_allowed(struct nvmet_req *req,
 813                 const char *hostnqn)
 814 {
 815         struct nvmet_subsys_link *s;
 816
 817         list_for_each_entry(s, &req->port->subsystems, entry) {
 818                 if (__nvmet_host_allowed(s->subsys, hostnqn))
 819                         return true;
 820         }
 821
 822         return false;
 823 }
 824
 825 bool nvmet_host_allowed(struct nvmet_req *req, struct nvmet_subsys *subsys,
 826                 const char *hostnqn)
 827 {
 828         lockdep_assert_held(&nvmet_config_sem);
 829
 830         if (subsys->type == NVME_NQN_DISC)
 831                 return nvmet_host_discovery_allowed(req, hostnqn);
 832         else
 833                 return __nvmet_host_allowed(subsys, hostnqn);
 834 }
 835
 836 u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 837                 struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp)
 838 {
 839         struct nvmet_subsys *subsys;
 840         struct nvmet_ctrl *ctrl;
 841         int ret;
 842         u16 status;
 843
 844         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 845         subsys = nvmet_find_get_subsys(req->port, subsysnqn);
 846         if (!subsys) {
 847                 pr_warn("connect request for invalid subsystem %s!\n",
 848                         subsysnqn);
 849                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 850                 goto out;
 851         }
 852
 853         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 854         down_read(&nvmet_config_sem);
 855         if (!nvmet_host_allowed(req, subsys, hostnqn)) {
 856                 pr_info("connect by host %s for subsystem %s not allowed\n",
 857                         hostnqn, subsysnqn);
 858                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn);
 859                 up_read(&nvmet_config_sem);
 860                 status = NVME_SC_CONNECT_INVALID_HOST | NVME_SC_DNR;
 861                 goto out_put_subsystem;
 862         }
 863         up_read(&nvmet_config_sem);
 864
 865         status = NVME_SC_INTERNAL;
 866         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
 867         if (!ctrl)
 868                 goto out_put_subsystem;
 869         mutex_init(&ctrl->lock);
 870
 871         nvmet_init_cap(ctrl);
 872
 873         INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);
 874         INIT_LIST_HEAD(&ctrl->async_events);
 875
 876         memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);
 877         memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);
 878
 879         kref_init(&ctrl->ref);
 880         ctrl->subsys = subsys;
 881         WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL);
 882
 883         ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES,
 884                         sizeof(__le32), GFP_KERNEL);
 885         if (!ctrl->changed_ns_list)
 886                 goto out_free_ctrl;
 887
 888         ctrl->cqs = kcalloc(subsys->max_qid + 1,
 889                         sizeof(struct nvmet_cq *),
 890                         GFP_KERNEL);
 891         if (!ctrl->cqs)
 892                 goto out_free_changed_ns_list;
 893
 894         ctrl->sqs = kcalloc(subsys->max_qid + 1,
 895                         sizeof(struct nvmet_sq *),
 896                         GFP_KERNEL);
 897         if (!ctrl->sqs)
 898                 goto out_free_cqs;
 899
 900         ret = ida_simple_get(&cntlid_ida,
 901                              NVME_CNTLID_MIN, NVME_CNTLID_MAX,
 902                              GFP_KERNEL);
 903         if (ret < 0) {
 904                 status = NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR;
 905                 goto out_free_sqs;
 906         }
 907         ctrl->cntlid = ret;
 908
 909         ctrl->ops = req->ops;
 910         if (ctrl->subsys->type == NVME_NQN_DISC) {
 911                 /* Don't accept keep-alive timeout for discovery controllers */
 912                 if (kato) {
 913                         status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 914                         goto out_remove_ida;
 915                 }
 916
 917                 /*
 918                  * Discovery controllers use some arbitrary high value in order
 919                  * to cleanup stale discovery sessions
 920                  *
 921                  * From the latest base diff RC:
 922                  * "The Keep Alive command is not supported by
 923                  * Discovery controllers. A transport may specify a
 924                  * fixed Discovery controller activity timeout value
 925                  * (e.g., 2 minutes).  If no commands are received
 926                  * by a Discovery controller within that time
 927                  * period, the controller may perform the
 928                  * actions for Keep Alive Timer expiration".
 929                  */
 930                 ctrl->kato = NVMET_DISC_KATO;
 931         } else {
 932                 /* keep-alive timeout in seconds */
 933                 ctrl->kato = DIV_ROUND_UP(kato, 1000);
 934         }
 935         nvmet_start_keep_alive_timer(ctrl);
 936
 937         mutex_lock(&subsys->lock);
 938         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
 939         mutex_unlock(&subsys->lock);
 940
 941         *ctrlp = ctrl;
 942         return 0;
 943
 944 out_remove_ida:
 945         ida_simple_remove(&cntlid_ida, ctrl->cntlid);
 946 out_free_sqs:
 947         kfree(ctrl->sqs);
 948 out_free_cqs:
 949         kfree(ctrl->cqs);
 950 out_free_changed_ns_list:
 951         kfree(ctrl->changed_ns_list);
 952 out_free_ctrl:
 953         kfree(ctrl);
 954 out_put_subsystem:
 955         nvmet_subsys_put(subsys);
 956 out:
 957         return status;
 958 }
 959
 960 static void nvmet_ctrl_free(struct kref *ref)
 961 {
 962         struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref);
 963         struct nvmet_subsys *subsys = ctrl->subsys;
 964
 965         mutex_lock(&subsys->lock);
 966         list_del(&ctrl->subsys_entry);
 967         mutex_unlock(&subsys->lock);
 968
 969         nvmet_stop_keep_alive_timer(ctrl);
 970
 971         flush_work(&ctrl->async_event_work);
 972         cancel_work_sync(&ctrl->fatal_err_work);
 973
 974         ida_simple_remove(&cntlid_ida, ctrl->cntlid);
 975
 976         kfree(ctrl->sqs);
 977         kfree(ctrl->cqs);
 978         kfree(ctrl->changed_ns_list);
 979         kfree(ctrl);
 980
 981         nvmet_subsys_put(subsys);
 982 }
 983
 984 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl)
 985 {
 986         kref_put(&ctrl->ref, nvmet_ctrl_free);
 987 }
 988
 989 static void nvmet_fatal_error_handler(struct work_struct *work)
 990 {
 991         struct nvmet_ctrl *ctrl =
 992                         container_of(work, struct nvmet_ctrl, fatal_err_work);
 993
 994         pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid);
 995         ctrl->ops->delete_ctrl(ctrl);
 996 }
 997
 998 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)
 999 {
1000         mutex_lock(&ctrl->lock);
1001         if (!(ctrl->csts & NVME_CSTS_CFS)) {
1002                 ctrl->csts |= NVME_CSTS_CFS;
1003                 INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler);
1004                 schedule_work(&ctrl->fatal_err_work);
1005         }
1006         mutex_unlock(&ctrl->lock);
1007 }
1008 EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error);
1009
1010 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
1011                 const char *subsysnqn)
1012 {
1013         struct nvmet_subsys_link *p;
1014
1015         if (!port)
1016                 return NULL;
1017
1018         if (!strncmp(NVME_DISC_SUBSYS_NAME, subsysnqn,
1019                         NVMF_NQN_SIZE)) {
1020                 if (!kref_get_unless_zero(&nvmet_disc_subsys->ref))
1021                         return NULL;
1022                 return nvmet_disc_subsys;
1023         }
1024
1025         down_read(&nvmet_config_sem);
1026         list_for_each_entry(p, &port->subsystems, entry) {
1027                 if (!strncmp(p->subsys->subsysnqn, subsysnqn,
1028                                 NVMF_NQN_SIZE)) {
1029                         if (!kref_get_unless_zero(&p->subsys->ref))
1030                                 break;
1031                         up_read(&nvmet_config_sem);
1032                         return p->subsys;
1033                 }
1034         }
1035         up_read(&nvmet_config_sem);
1036         return NULL;
1037 }
1038
1039 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn,
1040                 enum nvme_subsys_type type)
1041 {
1042         struct nvmet_subsys *subsys;
1043
1044         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
1045         if (!subsys)
1046                 return NULL;
1047
1048         subsys->ver = NVME_VS(1, 3, 0); /* NVMe 1.3.0 */
1049         /* generate a random serial number as our controllers are ephemeral: */
1050         get_random_bytes(&subsys->serial, sizeof(subsys->serial));
1051
1052         switch (type) {
1053         case NVME_NQN_NVME:
1054                 subsys->max_qid = NVMET_NR_QUEUES;
1055                 break;
1056         case NVME_NQN_DISC:
1057                 subsys->max_qid = 0;
1058                 break;
1059         default:
1060                 pr_err("%s: Unknown Subsystem type - %d\n", __func__, type);
1061                 kfree(subsys);
1062                 return NULL;
1063         }
1064         subsys->type = type;
1065         subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE,
1066                         GFP_KERNEL);
1067         if (!subsys->subsysnqn) {
1068                 kfree(subsys);
1069                 return NULL;
1070         }
1071
1072         kref_init(&subsys->ref);
1073
1074         mutex_init(&subsys->lock);
1075         INIT_LIST_HEAD(&subsys->namespaces);
1076         INIT_LIST_HEAD(&subsys->ctrls);
1077         INIT_LIST_HEAD(&subsys->hosts);
1078
1079         return subsys;
1080 }
1081
1082 static void nvmet_subsys_free(struct kref *ref)
1083 {
1084         struct nvmet_subsys *subsys =
1085                 container_of(ref, struct nvmet_subsys, ref);
1086
1087         WARN_ON_ONCE(!list_empty(&subsys->namespaces));
1088
1089         kfree(subsys->subsysnqn);
1090         kfree(subsys);
1091 }
1092
1093 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys)
1094 {
1095         struct nvmet_ctrl *ctrl;
1096
1097         mutex_lock(&subsys->lock);
1098         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
1099                 ctrl->ops->delete_ctrl(ctrl);
1100         mutex_unlock(&subsys->lock);
1101 }
1102
1103 void nvmet_subsys_put(struct nvmet_subsys *subsys)
1104 {
1105         kref_put(&subsys->ref, nvmet_subsys_free);
1106 }
1107
1108 static int __init nvmet_init(void)
1109 {
1110         int error;
1111
1112         error = nvmet_init_discovery();
1113         if (error)
1114                 goto out;
1115
1116         error = nvmet_init_configfs();
1117         if (error)
1118                 goto out_exit_discovery;
1119         return 0;
1120
1121 out_exit_discovery:
1122         nvmet_exit_discovery();
1123 out:
1124         return error;
1125 }
1126
1127 static void __exit nvmet_exit(void)
1128 {
1129         nvmet_exit_configfs();
1130         nvmet_exit_discovery();
1131         ida_destroy(&cntlid_ida);
1132
1133         BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024);
1134         BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024);
1135 }
1136
1137 module_init(nvmet_init);
1138 module_exit(nvmet_exit);
1139
1140 MODULE_LICENSE("GPL v2");